Dune cache: add enabled-except-user-rules setting (#10944)

Signed-off-by: Nicolás Ojeda Bär <n.oje.bar@gmail.com>
ocaml · Sep 22, 2024 · 1db1f45 · 1db1f45
1 parent d6f1a27
commit 1db1f45
Show file tree

Hide file tree

Showing 20 changed files with 97 additions and 52 deletions.
diff --git a/bin/common.ml b/bin/common.ml
@@ -1,5 +1,4 @@
 open Stdune
-open Dune_config
 open Dune_config_file
 module Console = Dune_console
 module Graph = Dune_graph.Graph
@@ -412,12 +411,12 @@ let shared_with_config_file =
     let doc =
       Printf.sprintf
         "Enable or disable Dune cache (%s). Default is `%s'."
-        (Arg.doc_alts_enum Config.Toggle.all)
-        (Config.Toggle.to_string Dune_config.default.cache_enabled)
+        (Arg.doc_alts_enum Dune_config.Cache.Toggle.all)
+        (Dune_config.Cache.Toggle.to_string Dune_config.default.cache_enabled)
     in
     Arg.(
       value
-      & opt (some (enum Config.Toggle.all)) None
+      & opt (some (enum Dune_config.Cache.Toggle.all)) None
       & info [ "cache" ] ~docs ~env:(Cmd.Env.info ~doc "DUNE_CACHE") ~doc)
   and+ cache_storage_mode =
     let doc =
@@ -584,7 +583,6 @@ module Builder = struct
     ; file_watcher : Dune_engine.Scheduler.Run.file_watcher
     ; workspace_config : Dune_rules.Workspace.Clflags.t
     ; cache_debug_flags : Dune_engine.Cache_debug_flags.t
-    ; cache_rules_default : bool
     ; report_errors_config : Dune_engine.Report_errors_config.t
     ; separate_error_messages : bool
     ; stop_on_first_error : bool
@@ -936,20 +934,6 @@ module Builder = struct
                useful for Dune developers to make Dune tests of the digest cache more \
                reproducible.")
     and+ cache_debug_flags = cache_debug_flags_term
-    and+ cache_rules_default =
-      let default =
-        Dune_lang.Toggle.of_bool !Dune_engine.Clflags.can_go_in_shared_cache_default
-      in
-      let doc =
-        Printf.sprintf
-          "Enable or disable caching rules (%s). Default is `%s'."
-          (Arg.doc_alts_enum Config.Toggle.all)
-          (Config.Toggle.to_string default)
-      in
-      Arg.(
-        value
-        & opt (enum Config.Toggle.all) default
-        & info [ "cache-rules" ] ~docs ~env:(Cmd.Env.info ~doc "DUNE_CACHE_RULES") ~doc)
     and+ report_errors_config =
       Arg.(
         value
@@ -1026,7 +1010,6 @@ module Builder = struct
         ; config_from_config_file
         }
     ; cache_debug_flags
-    ; cache_rules_default = Dune_lang.Toggle.enabled cache_rules_default
     ; report_errors_config
     ; separate_error_messages
     ; stop_on_first_error
@@ -1236,14 +1219,18 @@ let init (builder : Builder.t) =
   Dune_rules.Global.init ~capture_outputs:c.builder.capture_outputs;
   let cache_config =
     match config.cache_enabled with
-    | `Disabled -> Dune_cache.Config.Disabled
-    | `Enabled ->
+    | Disabled -> Dune_cache.Config.Disabled
+    | Enabled_except_user_rules | Enabled ->
       Enabled
         { storage_mode = Option.value config.cache_storage_mode ~default:Hardlink
         ; reproducibility_check = config.cache_reproducibility_check
         }
   in
-  Log.info [ Pp.textf "Shared cache: %s" (Config.Toggle.to_string config.cache_enabled) ];
+  Log.info
+    [ Pp.textf
+        "Shared cache: %s"
+        (Dune_config.Cache.Toggle.to_string config.cache_enabled)
+    ];
   Log.info
     [ Pp.textf
         "Shared cache location: %s"
@@ -1277,7 +1264,10 @@ let init (builder : Builder.t) =
   Dune_rules.Clflags.ignore_lock_dir := c.builder.ignore_lock_dir;
   Dune_rules.Clflags.on_missing_dune_project_file
   := if c.builder.require_dune_project_file then Error else Warn;
-  Dune_engine.Clflags.can_go_in_shared_cache_default := c.builder.cache_rules_default;
+  (Dune_engine.Clflags.can_go_in_shared_cache_default
+   := match config.cache_enabled with
+      | Disabled | Enabled_except_user_rules -> false
+      | Enabled -> true);
   Log.info
     [ Pp.textf
         "Workspace root: %s"

diff --git a/doc/caching.rst b/doc/caching.rst
@@ -15,13 +15,24 @@ or simply undoing some changes within the same workspace.
 Configuration
 =============
 
-For now, Dune cache is an opt-in feature. There are three ways to enable it.
-Choose the one that is more convenient for you:
+There are three ways to configure the Dune cache.  Choose the one that is more
+convenient for you:
 
-* Add ``(cache enabled)`` to your Dune configuration file
+* Add ``(cache <setting>)`` to your Dune configuration file
   (``~/.config/dune/config`` by default).
-* Set the environment variable ``DUNE_CACHE`` to ``enabled``
-* Run Dune with the ``--cache=enabled`` flag.
+* Set the environment variable ``DUNE_CACHE`` to ``<setting>``
+* Run Dune with the ``--cache=<setting>`` flag.
+
+Here, ``<setting>`` must be one of:
+
+* ``disabled``: disables the Dune cache completely.
+
+* ``enabled-except-user-rules``: enables the Dune cache, but excludes
+  user-written rules. This setting is a conservative choice that can avoid
+  breaking rules whose dependencies are not correctly specified. Currently the
+  default.
+
+* ``enabled``: enables the Dune cache unconditionally.
 
 By default, Dune stores the cache in your ``XDG_CACHE_HOME`` directory on \*nix
 systems and ``%LOCALAPPDATA%\Microsoft\Windows\Temporary Internet Files\dune`` on Windows.

diff --git a/doc/changes/10944.md b/doc/changes/10944.md
@@ -0,0 +1,4 @@
+- A new Dune cache setting: `enabled-except-user-rules`, which enables the Dune
+  cache, but excludes user-written rules from it. This is a conservative choice
+  that can avoid breaking rules whose dependencies are not correctly
+  specified. This is the current default. (#10944, @nojb)
diff --git a/doc/reference/config/cache.rst b/doc/reference/config/cache.rst
@@ -12,4 +12,8 @@ where ``<setting>`` is one of:
 
 - ``enabled`` enables Dune cache.
 
+- ``enabled-except-user-rules`` enables the Dune cache, but exclude user-written
+  rules. This setting is a conservative choice that can avoid breaking rules
+  whose dependencies are not correctly specified. Currently the default.
+
 - ``disabled`` disables Dune cache.
diff --git a/src/dune_config_file/dune_config_file.ml b/src/dune_config_file/dune_config_file.ml
@@ -111,6 +111,41 @@ module Dune_config = struct
   end
 
   module Cache = struct
+    module Toggle = struct
+      type t =
+        | Disabled
+        | Enabled_except_user_rules
+        | Enabled
+
+      let to_string = function
+        | Disabled -> "disabled"
+        | Enabled_except_user_rules -> "enabled-except-user-rules"
+        | Enabled -> "enabled"
+      ;;
+
+      let all =
+        List.map
+          ~f:(fun x -> to_string x, x)
+          [ Disabled; Enabled_except_user_rules; Enabled ]
+      ;;
+
+      let decode ~check =
+        let open Dune_lang.Decoder in
+        enum'
+          [ to_string Disabled, return Disabled
+          ; to_string Enabled, return Enabled
+          ; ( to_string Enabled_except_user_rules
+            , check (3, 17) >>> return Enabled_except_user_rules )
+          ]
+      ;;
+
+      let to_dyn = function
+        | Disabled -> Dyn.variant "Disabed" []
+        | Enabled_except_user_rules -> Dyn.variant "Enabled_except_user_rules" []
+        | Enabled -> Dyn.variant "Enabled" []
+      ;;
+    end
+
     module Transport_deprecated = struct
       type t =
         | Daemon
@@ -153,7 +188,7 @@ module Dune_config = struct
       ; concurrency : Concurrency.t field
       ; terminal_persistence : Terminal_persistence.t field
       ; sandboxing_preference : Sandboxing_preference.t field
-      ; cache_enabled : Config.Toggle.t field
+      ; cache_enabled : Cache.Toggle.t field
       ; cache_reproducibility_check : Dune_cache.Config.Reproducibility_check.t field
       ; cache_storage_mode : Cache.Storage_mode.t field
       ; action_stdout_on_success : Action_output_on_success.t field
@@ -220,7 +255,7 @@ module Dune_config = struct
         ; "terminal_persistence", field Terminal_persistence.to_dyn terminal_persistence
         ; ( "sandboxing_preference"
           , field (Dyn.list Sandbox_mode.to_dyn) sandboxing_preference )
-        ; "cache_enabled", field Config.Toggle.to_dyn cache_enabled
+        ; "cache_enabled", field Cache.Toggle.to_dyn cache_enabled
         ; ( "cache_reproducibility_check"
           , field
               Dune_cache.Config.Reproducibility_check.to_dyn
@@ -316,7 +351,7 @@ module Dune_config = struct
     ; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto)
     ; terminal_persistence = Clear_on_rebuild
     ; sandboxing_preference = []
-    ; cache_enabled = `Enabled
+    ; cache_enabled = Enabled_except_user_rules
     ; cache_reproducibility_check = Skip
     ; cache_storage_mode = Some (Dune_cache_storage.Mode.default ())
     ; action_stdout_on_success = Print
@@ -342,7 +377,7 @@ module Dune_config = struct
       field_o "terminal-persistence" (1, 0) Terminal_persistence.decode
     and+ sandboxing_preference =
       field_o "sandboxing_preference" (1, 0) Sandboxing_preference.decode
-    and+ cache_enabled = field_o "cache" (2, 0) (enum Config.Toggle.all)
+    and+ cache_enabled = field_o "cache" (2, 0) (Cache.Toggle.decode ~check)
     and+ _cache_transport_unused_since_3_0 =
       field_o
         "cache-transport"

diff --git a/src/dune_config_file/dune_config_file.mli b/src/dune_config_file/dune_config_file.mli
@@ -2,7 +2,6 @@ module Dune_config : sig
   (** Dune configuration (visible to the user) *)
 
   open Stdune
-  open Dune_config
   module Display : module type of Display
 
   module Project_defaults : sig
@@ -29,6 +28,21 @@ module Dune_config : sig
   end
 
   module Cache : sig
+    module Toggle : sig
+      type t =
+        | Disabled
+        | Enabled_except_user_rules
+        | Enabled
+
+      val all : (string * t) list
+
+      val decode
+        :  check:(Dune_lang.Syntax.Version.t -> unit Dune_lang.Decoder.t)
+        -> t Dune_lang.Decoder.t
+
+      val to_string : t -> string
+    end
+
     module Storage_mode : sig
       type t = Dune_cache_storage.Mode.t option
 
@@ -61,7 +75,7 @@ module Dune_config : sig
       ; concurrency : Concurrency.t field
       ; terminal_persistence : Terminal_persistence.t field
       ; sandboxing_preference : Sandboxing_preference.t field
-      ; cache_enabled : Config.Toggle.t field
+      ; cache_enabled : Cache.Toggle.t field
       ; cache_reproducibility_check : Dune_cache.Config.Reproducibility_check.t field
       ; cache_storage_mode : Cache.Storage_mode.t field
       ; action_stdout_on_success : Action_output_on_success.t field

diff --git a/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t b/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t
@@ -2,7 +2,6 @@ This checks what happens when a file available in the cache is used in a directo
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
   $ export DUNE_CACHE=enabled
-  $ export DUNE_CACHE_RULES=enabled
   $ . ./helpers.sh
 
   $ cat > dune-project << EOF

diff --git a/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t b/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t
@@ -1,7 +1,6 @@
 We create 2 directory targets which share a whole subdirectory.
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
   $ export DUNE_CACHE=enabled
   $ . ./helpers.sh
 

diff --git a/test/blackbox-tests/test-cases/directory-targets/cache.t b/test/blackbox-tests/test-cases/directory-targets/cache.t
@@ -1,7 +1,6 @@
 We test that directory targets can go in the shared cache. See #8067.
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
   $ export DUNE_CACHE=enabled
 
 In project a, we create a rule with a directory target. The script that creates

diff --git a/test/blackbox-tests/test-cases/dune-cache/config.t b/test/blackbox-tests/test-cases/dune-cache/config.t
@@ -26,7 +26,6 @@ Check that old cache configuration format works fine with an old language
 Test that DUNE_CACHE_ROOT can be used to control the cache location
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
 
 Build succeeds and the 'copy' mode is respected
 

diff --git a/test/blackbox-tests/test-cases/dune-cache/dedup.t b/test/blackbox-tests/test-cases/dune-cache/dedup.t
@@ -2,7 +2,6 @@ Test deduplication of build artifacts when using Dune cache with hard links.
 
   $ export DUNE_CACHE=enabled
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > dune-project <<EOF
   > (lang dune 2.1)

diff --git a/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t b/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t
@@ -1,7 +1,6 @@
 Check that Dune cache can cope with missing file/metadata entries.
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > config <<EOF
   > (lang dune 2.1)

diff --git a/test/blackbox-tests/test-cases/dune-cache/mode-copy.t b/test/blackbox-tests/test-cases/dune-cache/mode-copy.t
@@ -5,7 +5,6 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former.
 
   $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
   $ export XDG_CACHE_HOME=$PWD/.xdg-cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > config <<EOF
   > (lang dune 3.0)

diff --git a/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t b/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t
@@ -5,7 +5,6 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former.
 
   $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
   $ export XDG_CACHE_HOME=$PWD/.xdg-cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > config <<EOF
   > (lang dune 2.1)

diff --git a/test/blackbox-tests/test-cases/dune-cache/repro-check.t b/test/blackbox-tests/test-cases/dune-cache/repro-check.t
@@ -1,7 +1,6 @@
 Test reproducibility check
 
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
   $ cat > config <<EOF
   > (lang dune 3.0)
   > (cache enabled)

diff --git a/test/blackbox-tests/test-cases/dune-cache/size.t/run.t b/test/blackbox-tests/test-cases/dune-cache/size.t/run.t
@@ -3,7 +3,6 @@ the cache.
 
   $ export DUNE_CACHE=enabled
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > config << EOF
   > (lang dune 3.7)

diff --git a/test/blackbox-tests/test-cases/dune-cache/symlink.t b/test/blackbox-tests/test-cases/dune-cache/symlink.t
@@ -3,7 +3,6 @@ produced symbolic links work correctly and are appropriately cached.
 
   $ export DUNE_CACHE=enabled
   $ export DUNE_CACHE_ROOT=$PWD/.cache
-  $ export DUNE_CACHE_RULES=enabled
 
   $ cat > dune-project <<EOF
   > (lang dune 2.1)

diff --git a/test/blackbox-tests/test-cases/dune-cache/trim.t b/test/blackbox-tests/test-cases/dune-cache/trim.t
@@ -1,5 +1,4 @@
   $ export DUNE_CACHE=enabled
-  $ export DUNE_CACHE_RULES=enabled
   $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
   $ export XDG_CACHE_HOME=$PWD/.xdg-cache
 

diff --git a/test/blackbox-tests/test-cases/pkg/fetch-cache.t b/test/blackbox-tests/test-cases/pkg/fetch-cache.t
@@ -2,13 +2,12 @@ Testing that files are only fetched once.
 
   $ . ./helpers.sh
 
-No need to set DUNE_CACHE (enabled by default) nor DUNE_CACHE_RULES as the
+No need to set DUNE_CACHE (enabled by default) as the
 fetch rules are always considered safe to cache, but we'll set a custom
 directory for the shared cache.
 
   $ export DUNE_CACHE_ROOT=$(pwd)/dune-cache
   $ unset DUNE_CACHE
-  $ unset DUNE_CACHE_RULES
 
 Set up a project that depends on a package that is being downloaded
 

diff --git a/test/expect-tests/dune_config_file/dune_config_test.ml b/test/expect-tests/dune_config_file/dune_config_test.ml
@@ -22,7 +22,7 @@ let%expect_test "cache-check-probability 0.1" =
     ; concurrency = Fixed 1
     ; terminal_persistence = Clear_on_rebuild
     ; sandboxing_preference = []
-    ; cache_enabled = Enabled
+    ; cache_enabled = Enabled_except_user_rules
     ; cache_reproducibility_check = Check_with_probability 0.1
     ; cache_storage_mode = Some Hardlink
     ; action_stdout_on_success = Print
@@ -45,7 +45,7 @@ let%expect_test "cache-storage-mode copy" =
     ; concurrency = Fixed 1
     ; terminal_persistence = Clear_on_rebuild
     ; sandboxing_preference = []
-    ; cache_enabled = Enabled
+    ; cache_enabled = Enabled_except_user_rules
     ; cache_reproducibility_check = Skip
     ; cache_storage_mode = Some Copy
     ; action_stdout_on_success = Print
@@ -68,7 +68,7 @@ let%expect_test "cache-storage-mode hardlink" =
     ; concurrency = Fixed 1
     ; terminal_persistence = Clear_on_rebuild
     ; sandboxing_preference = []
-    ; cache_enabled = Enabled
+    ; cache_enabled = Enabled_except_user_rules
     ; cache_reproducibility_check = Skip
     ; cache_storage_mode = Some Hardlink
     ; action_stdout_on_success = Print