Merge branch 'main' into lang-dune-parsing-warning

.gitignore

@@ -18,3 +18,5 @@ _perf
 dune.exe
 Makefile.dev
 src/dune/setup.ml
+
+.DS_Store

CHANGES.md

@@ -5,6 +5,15 @@ Unreleased
   dune 2.3suffix)` were silently parsed as `2.3` and we know suggest to remove
   the prefix. (#5040, @emillon)
 
+- In watch mode, use fsevents instead of fswatch on OSX (#4937, #4990, fixes
+  #4896 @rgrinberg)
+
+- Remove `inotifywait` watch mode backend on Linux. We now use the inotify API
+  exclusively (#4941, @rgrinberg)
+
+- Report cycles between virtual libraries and their implementation (#5050,
+  fixes #2896, @rgrinberg)
+
 - Allow users to specify dynamic dependencies in rules. For example `(deps
   %{read:foo.gen})` (#4662, fixes #4089, @jeremiedimino)
 

bin/arg.ml

@@ -71,7 +71,7 @@ module Dep = struct
              ~mode:Dune_lang.Parser.Mode.Single s)
       with
       | x -> `Ok x
-      | exception User_error.E (msg, _) -> `Error (User_message.to_string msg))
+      | exception User_error.E msg -> `Error (User_message.to_string msg))
 
   let string_of_alias ~recursive sv =
     let prefix =
@@ -115,14 +115,17 @@ let bytes =
       Dune_lang.Decoder.parse Dune_lang.Decoder.bytes_unit Univ_map.empty ast
     with
     | x -> Result.Ok x
-    | exception User_error.E (msg, _) ->
+    | exception User_error.E msg ->
       Result.Error (`Msg (User_message.to_string msg))
   in
   let pp_print_int64 state i =
     Format.pp_print_string state (Int64.to_string i)
   in
   conv (decode, pp_print_int64)
 
+let graph_format : Dune_graph.Graph.File_format.t conv =
+  conv Dune_graph.Graph.File_format.conv
+
 let context_name : Context_name.t conv = conv Context_name.conv
 
 let lib_name = conv Dune_engine.Lib_name.conv

bin/arg.mli

@@ -32,6 +32,8 @@ val context_name : Context_name.t conv
 
 val dep : Dep.t conv
 
+val graph_format : Dune_graph.Graph.File_format.t conv
+
 val path : Path.t conv
 
 val package_name : Package.Name.t conv

bin/build_cmd.ml

@@ -24,12 +24,9 @@ let with_metrics ~common f =
       Fiber.return ())
 
 let run_build_system ~common ~request =
-  let run ~(request : unit Action_builder.t) =
+  let run ~(toplevel : unit Memo.Lazy.t) =
     with_metrics ~common (fun () ->
-        Build_system.run (fun () ->
-            let open Memo.Build.O in
-            let+ (), _facts = Action_builder.run request Eager in
-            ()))
+        Build_system.run (fun () -> Memo.Lazy.force toplevel))
   in
   let open Fiber.O in
   Fiber.finalize
@@ -47,7 +44,35 @@ let run_build_system ~common ~request =
         Action_builder.bind (Action_builder.memo_build setup) ~f:(fun setup ->
             request setup)
       in
-      run ~request)
+      (* CR-someday cmoseley: Can we avoid creating a new lazy memo node every
+         time the build system is rerun? *)
+      (* This top-level node is used for traversing the whole Memo graph. *)
+      let toplevel_cell, toplevel =
+        Memo.Lazy.Expert.create ~name:"toplevel" (fun () ->
+            let open Memo.Build.O in
+            let+ (), (_ : Dep.Fact.t Dep.Map.t) =
+              Action_builder.run request Eager
+            in
+            ())
+      in
+      let* res = run ~toplevel in
+      let+ () =
+        match Common.dump_memo_graph_file common with
+        | None -> Fiber.return ()
+        | Some file ->
+          let path = Path.of_filename_relative_to_initial_cwd file in
+          let+ graph =
+            Memo.dump_cached_graph
+              ~time_nodes:(Common.dump_memo_graph_with_timing common)
+              toplevel_cell
+          in
+          Graph.serialize graph ~path
+            ~format:(Common.dump_memo_graph_format common)
+        (* CR-someday cmoseley: It would be nice to use Persistent to dump a
+           copy of the graph's internal representation here, so it could be used
+           without needing to re-run the build*)
+      in
+      res)
     ~finally:(fun () ->
       Hooks.End_of_build.run ();
       Fiber.return ())

bin/clean.ml

@@ -18,7 +18,7 @@ let command =
        useless but with some FS this also causes [dune clean] to fail (cf
        https://github.com/ocaml/dune/issues/2964). *)
     let _config = Common.init common ~log_file:No_log_file in
-    Build_system.files_in_source_tree_to_delete ()
+    Dune_engine.Target_promotion.files_in_source_tree_to_delete ()
     |> Path.Set.iter ~f:Path.unlink_no_err;
     Path.rm_rf Path.build_dir
   in

bin/common.ml

@@ -2,6 +2,7 @@ open Stdune
 module Config = Dune_util.Config
 module Colors = Dune_rules.Colors
 module Clflags = Dune_engine.Clflags
+module Graph = Dune_graph.Graph
 module Package = Dune_engine.Package
 module Profile = Dune_rules.Profile
 module Term = Cmdliner.Term
@@ -39,6 +40,9 @@ type t =
   ; default_target : Arg.Dep.t (* For build & runtest only *)
   ; watch : Dune_engine.Watch_mode_config.t
   ; print_metrics : bool
+  ; dump_memo_graph_file : string option
+  ; dump_memo_graph_format : Graph.File_format.t
+  ; dump_memo_graph_with_timing : bool
   ; stats_trace_file : string option
   ; always_show_command_line : bool
   ; promote_install_files : bool
@@ -57,6 +61,12 @@ let watch t = t.watch
 
 let print_metrics t = t.print_metrics
 
+let dump_memo_graph_file t = t.dump_memo_graph_file
+
+let dump_memo_graph_format t = t.dump_memo_graph_format
+
+let dump_memo_graph_with_timing t = t.dump_memo_graph_with_timing
+
 let file_watcher t = t.file_watcher
 
 let default_target t = t.default_target
@@ -831,6 +841,31 @@ let term ~default_root_is_cwd =
       value & flag
       & info [ "print-metrics" ] ~docs
           ~doc:"Print out various performance metrics after every build")
+  and+ dump_memo_graph_file =
+    Arg.(
+      value
+      & opt (some string) None
+      & info [ "dump-memo-graph" ] ~docs ~docv:"FILE"
+          ~doc:
+            "Dumps the dependency graph to a file after the build is complete")
+  and+ dump_memo_graph_format =
+    Arg.(
+      value & opt graph_format Gexf
+      & info
+          [ "dump-memo-graph-format" ]
+          ~docs ~docv:"FORMAT"
+          ~doc:"File format to be used when dumping dependency graph")
+  and+ dump_memo_graph_with_timing =
+    Arg.(
+      value & flag
+      & info
+          [ "dump-memo-graph-with-timing" ]
+          ~docs
+          ~doc:
+            "With $(b,--dump-memo-graph), will re-run each cached node in the \
+             Memo graph after building and include the runtime in the output. \
+             Since all nodes contain a cached value, this will measure just \
+             the runtime of each node")
   and+ { Options_implied_by_dash_p.root
        ; only_packages
        ; ignore_promoted_rules
@@ -988,6 +1023,9 @@ let term ~default_root_is_cwd =
   ; default_target
   ; watch
   ; print_metrics
+  ; dump_memo_graph_file
+  ; dump_memo_graph_format
+  ; dump_memo_graph_with_timing
   ; stats_trace_file
   ; always_show_command_line
   ; promote_install_files

bin/common.mli

@@ -10,6 +10,12 @@ val stats : t -> Dune_stats.t option
 
 val print_metrics : t -> bool
 
+val dump_memo_graph_file : t -> string option
+
+val dump_memo_graph_format : t -> Dune_graph.Graph.File_format.t
+
+val dump_memo_graph_with_timing : t -> bool
+
 val watch : t -> Dune_engine.Watch_mode_config.t
 
 val file_watcher : t -> Dune_engine.Scheduler.Run.file_watcher

bin/dune

@@ -8,6 +8,7 @@
   unix
   dune_cache
   dune_cache_storage
+  dune_graph
   dune_rules
   dune_engine
   dune_util

bin/import.ml

@@ -20,7 +20,7 @@ module Dpath = Dune_engine.Dpath
 module Install = Dune_engine.Install
 module Section = Dune_engine.Section
 module Watermarks = Dune_rules.Watermarks
-module Promotion = Dune_engine.Promotion
+module Diff_promotion = Dune_engine.Diff_promotion
 module Colors = Dune_rules.Colors
 module Dune_project = Dune_engine.Dune_project
 module Workspace = Dune_rules.Workspace
@@ -29,6 +29,7 @@ module Targets = Dune_engine.Targets
 module Profile = Dune_rules.Profile
 module Log = Dune_util.Log
 module Dune_rpc = Dune_rpc_private
+module Graph = Dune_graph.Graph
 include Common.Let_syntax
 
 let in_group (t, info) = (Term.Group.Term t, info)
@@ -79,6 +80,7 @@ module Scheduler = struct
     | Scheduler.Run.Event.Tick -> Console.Status_line.refresh ()
     | Source_files_changed { details_hum } ->
       maybe_clear_screen ~details_hum dune_config
+    | Skipped_restart -> ()
     | Build_interrupted ->
       Console.Status_line.set
         (Live

bin/print_rules.ml

@@ -40,7 +40,7 @@ let print_rule_makefile ppf (rule : Dune_engine.Reflection.Rule.t) =
         Path.Build.Set.union files dirs)
   in
   Format.fprintf ppf
-    "@[<hov 2>@{<makefile-stuff>%a:%t@}@]@,@<0>\t@{<makefile-action>%a@}@,@,"
+    "@[<hov 2>@{<makefile-stuff>%a:%t@}@]@,@<0>\t@{<makefile-action>%a@}\n"
     (Format.pp_print_list ~pp_sep:Format.pp_print_space (fun ppf p ->
          Format.pp_print_string ppf (Path.to_string p)))
     (List.map ~f:Path.build (Path.Build.Set.to_list targets))

bin/promote.ml

@@ -23,7 +23,7 @@ let command =
       Arg.(value & pos_all Cmdliner.Arg.file [] & info [] ~docv:"FILE")
     in
     let _config = Common.init common in
-    Promotion.promote_files_registered_in_last_run
+    Diff_promotion.promote_files_registered_in_last_run
       (match files with
       | [] -> All
       | _ ->

boot/libs.ml

@@ -9,6 +9,7 @@ let local_libraries =
     Some "Dune_filesystem_stubs", false, None)
   ; ("vendor/csexp/src", Some "Csexp", false, None)
   ; ("otherlibs/stdune-unstable", Some "Stdune", false, None)
+  ; ("src/dune_graph", Some "Dune_graph", false, None)
   ; ("src/dune_lang", Some "Dune_lang", false, None)
   ; ("vendor/incremental-cycles/src", Some "Incremental_cycles", false, None)
   ; ("src/dag", Some "Dag", false, None)

otherlibs/action-plugin/test/depend-on-directory-without-targets/run.t

@@ -18,5 +18,4 @@
   $ cp ./bin/foo.exe ./
 
   $ dune runtest
-           foo alias runtest
   Directory listing: [some_file1; some_file2]

otherlibs/action-plugin/test/dependency-rebuilt-but-not-changed/run.t

@@ -25,7 +25,6 @@ they were forced to rebuild.
 
   $ dune runtest
   Building some_file!
-           foo alias runtest
   Hello from some_file!
 
   $ dune runtest

otherlibs/action-plugin/test/depends-on-directory-with-glob/run.t

@@ -39,6 +39,5 @@
   $ dune runtest
   Building some_file!
   Building some_file_but_different!
-           foo alias runtest
   some_file
   some_file_but_different

otherlibs/action-plugin/test/do-not-rebuild-unneeded-dependency/run.t

@@ -45,7 +45,6 @@ only the dependencies up to this stage are rebuilt.
   $ dune runtest
   Building foo_or_bar!
   Building foo!
-        client alias runtest
   Hello from foo!
 
   $ printf "bar" > foo_or_bar_source
@@ -55,5 +54,4 @@ only the dependencies up to this stage are rebuilt.
   $ dune runtest
   Building foo_or_bar!
   Building bar!
-        client alias runtest
   Hello from bar!

otherlibs/action-plugin/test/no-dependencies/run.t

@@ -16,5 +16,4 @@ but requires no dependencies can be successfully run.
   $ cp ./bin/foo.exe ./
 
   $ dune runtest
-           foo alias runtest
   Hello from foo!

otherlibs/action-plugin/test/one-dependency-with-chdir/run.t

@@ -25,5 +25,4 @@ when we 'chdir' into different directory.
 
   $ cp ./bin/foo.exe ./some_dir
   $ dune runtest
-           foo alias runtest
   Hello from some_dependency!

otherlibs/action-plugin/test/one-dependency/run.t

@@ -19,5 +19,4 @@ and requires one dependency can be successfully run.
   $ cp ./bin/foo.exe ./
 
   $ dune runtest
-           foo alias runtest
   Hello from some_dependency!

otherlibs/action-plugin/test/one-directory-dependency/run.t

@@ -32,7 +32,6 @@ directory to be build.
   $ cp ./bin/foo.exe ./
 
   $ dune runtest
-           foo alias runtest
   dune
   some_file1
   some_file2

otherlibs/action-plugin/test/one-undeclared-target/run.t

@@ -16,7 +16,5 @@
   1 | (rule
   2 |  (alias runtest)
   3 |  (action (dynamic-run ./foo.exe)))
-           foo alias runtest (exit 1)
-  (cd _build/default && ./foo.exe)
   bar is written despite not being declared as a target in dune file. To fix, add it to target list in dune file.
   [1]

otherlibs/action-plugin/test/ordinary-executable/run.t

@@ -15,7 +15,6 @@ ordinary executable instead of one linked against dune-action-plugin.
   $ cp ./bin/foo.exe ./
 
   $ dune runtest
-           foo alias runtest
   Hello from foo!
   File "dune", line 1, characters 0-57:
   1 | (rule

otherlibs/action-plugin/test/two-stages-dependency-choose/run.t

@@ -34,5 +34,4 @@ on based on dependency from the previous stage.
 
   $ dune runtest
   Building bar!
-        client alias runtest
   Hello from bar!

otherlibs/site/src/plugins/meta_parser.ml

@@ -27,13 +27,13 @@ module Meta_parser = Dune_meta_parser.Meta_parser.Make (struct
     module Style = struct
       type t = unit
     end
-  end
 
-  module User_error = struct
     module Annot = struct
       type t = unit
     end
+  end
 
+  module User_error = struct
     let raise ?loc:_ ?hints:_ ?annots:_ texts =
       invalid_arg (String.concat " " texts)
   end

otherlibs/site/test/run.t

@@ -358,7 +358,6 @@ Test compiling an external plugin
   run c: registered:e,b.
 
   $ OCAMLPATH=_install/lib:$OCAMLPATH dune build @runtest
-             c alias e/runtest
   run a
   a: $TESTCASE_ROOT/_build/install/default/share/a/data
   run c: a linked registered:.

otherlibs/stdune-unstable/env.ml

@@ -77,9 +77,17 @@ let add t ~var ~value = make (Map.set t.vars var value)
 
 let remove t ~var = make (Map.remove t.vars var)
 
-let extend t ~vars = make (Map.superpose t.vars vars)
-
-let extend_env x y = extend x ~vars:y.vars
+let extend t ~vars =
+  if Map.is_empty vars then
+    t
+  else
+    make (Map.superpose t.vars vars)
+
+let extend_env x y =
+  if Map.is_empty x.vars then
+    y
+  else
+    extend x ~vars:y.vars
 
 let to_dyn t =
   let open Dyn.Encoder in