From 1db1f45f9d6c111665a8af110dfa10606732cffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Ojeda=20B=C3=A4r?= Date: Sun, 22 Sep 2024 13:49:00 +0200 Subject: [PATCH] Dune cache: add `enabled-except-user-rules` setting (#10944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolás Ojeda Bär --- bin/common.ml | 38 ++++++---------- doc/caching.rst | 21 ++++++--- doc/changes/10944.md | 4 ++ doc/reference/config/cache.rst | 4 ++ src/dune_config_file/dune_config_file.ml | 43 +++++++++++++++++-- src/dune_config_file/dune_config_file.mli | 18 +++++++- .../directory-targets/cache-file-and-dir.t | 1 - .../directory-targets/cache-shared-subdir.t | 1 - .../test-cases/directory-targets/cache.t | 1 - .../test-cases/dune-cache/config.t | 1 - .../test-cases/dune-cache/dedup.t | 1 - .../dune-cache/missing-cache-entries.t | 1 - .../test-cases/dune-cache/mode-copy.t | 1 - .../test-cases/dune-cache/mode-hardlink.t | 1 - .../test-cases/dune-cache/repro-check.t | 1 - .../test-cases/dune-cache/size.t/run.t | 1 - .../test-cases/dune-cache/symlink.t | 1 - .../test-cases/dune-cache/trim.t | 1 - .../test-cases/pkg/fetch-cache.t | 3 +- .../dune_config_file/dune_config_test.ml | 6 +-- 20 files changed, 97 insertions(+), 52 deletions(-) create mode 100644 doc/changes/10944.md diff --git a/bin/common.ml b/bin/common.ml index 0d14be50204..589e4769b68 100644 --- a/bin/common.ml +++ b/bin/common.ml @@ -1,5 +1,4 @@ open Stdune -open Dune_config open Dune_config_file module Console = Dune_console module Graph = Dune_graph.Graph @@ -412,12 +411,12 @@ let shared_with_config_file = let doc = Printf.sprintf "Enable or disable Dune cache (%s). Default is `%s'." - (Arg.doc_alts_enum Config.Toggle.all) - (Config.Toggle.to_string Dune_config.default.cache_enabled) + (Arg.doc_alts_enum Dune_config.Cache.Toggle.all) + (Dune_config.Cache.Toggle.to_string Dune_config.default.cache_enabled) in Arg.( value - & opt (some (enum Config.Toggle.all)) None + & opt (some (enum Dune_config.Cache.Toggle.all)) None & info [ "cache" ] ~docs ~env:(Cmd.Env.info ~doc "DUNE_CACHE") ~doc) and+ cache_storage_mode = let doc = @@ -584,7 +583,6 @@ module Builder = struct ; file_watcher : Dune_engine.Scheduler.Run.file_watcher ; workspace_config : Dune_rules.Workspace.Clflags.t ; cache_debug_flags : Dune_engine.Cache_debug_flags.t - ; cache_rules_default : bool ; report_errors_config : Dune_engine.Report_errors_config.t ; separate_error_messages : bool ; stop_on_first_error : bool @@ -936,20 +934,6 @@ module Builder = struct useful for Dune developers to make Dune tests of the digest cache more \ reproducible.") and+ cache_debug_flags = cache_debug_flags_term - and+ cache_rules_default = - let default = - Dune_lang.Toggle.of_bool !Dune_engine.Clflags.can_go_in_shared_cache_default - in - let doc = - Printf.sprintf - "Enable or disable caching rules (%s). Default is `%s'." - (Arg.doc_alts_enum Config.Toggle.all) - (Config.Toggle.to_string default) - in - Arg.( - value - & opt (enum Config.Toggle.all) default - & info [ "cache-rules" ] ~docs ~env:(Cmd.Env.info ~doc "DUNE_CACHE_RULES") ~doc) and+ report_errors_config = Arg.( value @@ -1026,7 +1010,6 @@ module Builder = struct ; config_from_config_file } ; cache_debug_flags - ; cache_rules_default = Dune_lang.Toggle.enabled cache_rules_default ; report_errors_config ; separate_error_messages ; stop_on_first_error @@ -1236,14 +1219,18 @@ let init (builder : Builder.t) = Dune_rules.Global.init ~capture_outputs:c.builder.capture_outputs; let cache_config = match config.cache_enabled with - | `Disabled -> Dune_cache.Config.Disabled - | `Enabled -> + | Disabled -> Dune_cache.Config.Disabled + | Enabled_except_user_rules | Enabled -> Enabled { storage_mode = Option.value config.cache_storage_mode ~default:Hardlink ; reproducibility_check = config.cache_reproducibility_check } in - Log.info [ Pp.textf "Shared cache: %s" (Config.Toggle.to_string config.cache_enabled) ]; + Log.info + [ Pp.textf + "Shared cache: %s" + (Dune_config.Cache.Toggle.to_string config.cache_enabled) + ]; Log.info [ Pp.textf "Shared cache location: %s" @@ -1277,7 +1264,10 @@ let init (builder : Builder.t) = Dune_rules.Clflags.ignore_lock_dir := c.builder.ignore_lock_dir; Dune_rules.Clflags.on_missing_dune_project_file := if c.builder.require_dune_project_file then Error else Warn; - Dune_engine.Clflags.can_go_in_shared_cache_default := c.builder.cache_rules_default; + (Dune_engine.Clflags.can_go_in_shared_cache_default + := match config.cache_enabled with + | Disabled | Enabled_except_user_rules -> false + | Enabled -> true); Log.info [ Pp.textf "Workspace root: %s" diff --git a/doc/caching.rst b/doc/caching.rst index 0f0af9f5c8b..9278e10a4e6 100644 --- a/doc/caching.rst +++ b/doc/caching.rst @@ -15,13 +15,24 @@ or simply undoing some changes within the same workspace. Configuration ============= -For now, Dune cache is an opt-in feature. There are three ways to enable it. -Choose the one that is more convenient for you: +There are three ways to configure the Dune cache. Choose the one that is more +convenient for you: -* Add ``(cache enabled)`` to your Dune configuration file +* Add ``(cache )`` to your Dune configuration file (``~/.config/dune/config`` by default). -* Set the environment variable ``DUNE_CACHE`` to ``enabled`` -* Run Dune with the ``--cache=enabled`` flag. +* Set the environment variable ``DUNE_CACHE`` to ```` +* Run Dune with the ``--cache=`` flag. + +Here, ```` must be one of: + +* ``disabled``: disables the Dune cache completely. + +* ``enabled-except-user-rules``: enables the Dune cache, but excludes + user-written rules. This setting is a conservative choice that can avoid + breaking rules whose dependencies are not correctly specified. Currently the + default. + +* ``enabled``: enables the Dune cache unconditionally. By default, Dune stores the cache in your ``XDG_CACHE_HOME`` directory on \*nix systems and ``%LOCALAPPDATA%\Microsoft\Windows\Temporary Internet Files\dune`` on Windows. diff --git a/doc/changes/10944.md b/doc/changes/10944.md new file mode 100644 index 00000000000..73073d2f839 --- /dev/null +++ b/doc/changes/10944.md @@ -0,0 +1,4 @@ +- A new Dune cache setting: `enabled-except-user-rules`, which enables the Dune + cache, but excludes user-written rules from it. This is a conservative choice + that can avoid breaking rules whose dependencies are not correctly + specified. This is the current default. (#10944, @nojb) diff --git a/doc/reference/config/cache.rst b/doc/reference/config/cache.rst index db75dca0411..b4388f11e3f 100644 --- a/doc/reference/config/cache.rst +++ b/doc/reference/config/cache.rst @@ -12,4 +12,8 @@ where ```` is one of: - ``enabled`` enables Dune cache. +- ``enabled-except-user-rules`` enables the Dune cache, but exclude user-written + rules. This setting is a conservative choice that can avoid breaking rules + whose dependencies are not correctly specified. Currently the default. + - ``disabled`` disables Dune cache. diff --git a/src/dune_config_file/dune_config_file.ml b/src/dune_config_file/dune_config_file.ml index a2901197ebb..7d97cfdf90c 100644 --- a/src/dune_config_file/dune_config_file.ml +++ b/src/dune_config_file/dune_config_file.ml @@ -111,6 +111,41 @@ module Dune_config = struct end module Cache = struct + module Toggle = struct + type t = + | Disabled + | Enabled_except_user_rules + | Enabled + + let to_string = function + | Disabled -> "disabled" + | Enabled_except_user_rules -> "enabled-except-user-rules" + | Enabled -> "enabled" + ;; + + let all = + List.map + ~f:(fun x -> to_string x, x) + [ Disabled; Enabled_except_user_rules; Enabled ] + ;; + + let decode ~check = + let open Dune_lang.Decoder in + enum' + [ to_string Disabled, return Disabled + ; to_string Enabled, return Enabled + ; ( to_string Enabled_except_user_rules + , check (3, 17) >>> return Enabled_except_user_rules ) + ] + ;; + + let to_dyn = function + | Disabled -> Dyn.variant "Disabed" [] + | Enabled_except_user_rules -> Dyn.variant "Enabled_except_user_rules" [] + | Enabled -> Dyn.variant "Enabled" [] + ;; + end + module Transport_deprecated = struct type t = | Daemon @@ -153,7 +188,7 @@ module Dune_config = struct ; concurrency : Concurrency.t field ; terminal_persistence : Terminal_persistence.t field ; sandboxing_preference : Sandboxing_preference.t field - ; cache_enabled : Config.Toggle.t field + ; cache_enabled : Cache.Toggle.t field ; cache_reproducibility_check : Dune_cache.Config.Reproducibility_check.t field ; cache_storage_mode : Cache.Storage_mode.t field ; action_stdout_on_success : Action_output_on_success.t field @@ -220,7 +255,7 @@ module Dune_config = struct ; "terminal_persistence", field Terminal_persistence.to_dyn terminal_persistence ; ( "sandboxing_preference" , field (Dyn.list Sandbox_mode.to_dyn) sandboxing_preference ) - ; "cache_enabled", field Config.Toggle.to_dyn cache_enabled + ; "cache_enabled", field Cache.Toggle.to_dyn cache_enabled ; ( "cache_reproducibility_check" , field Dune_cache.Config.Reproducibility_check.to_dyn @@ -316,7 +351,7 @@ module Dune_config = struct ; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto) ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] - ; cache_enabled = `Enabled + ; cache_enabled = Enabled_except_user_rules ; cache_reproducibility_check = Skip ; cache_storage_mode = Some (Dune_cache_storage.Mode.default ()) ; action_stdout_on_success = Print @@ -342,7 +377,7 @@ module Dune_config = struct field_o "terminal-persistence" (1, 0) Terminal_persistence.decode and+ sandboxing_preference = field_o "sandboxing_preference" (1, 0) Sandboxing_preference.decode - and+ cache_enabled = field_o "cache" (2, 0) (enum Config.Toggle.all) + and+ cache_enabled = field_o "cache" (2, 0) (Cache.Toggle.decode ~check) and+ _cache_transport_unused_since_3_0 = field_o "cache-transport" diff --git a/src/dune_config_file/dune_config_file.mli b/src/dune_config_file/dune_config_file.mli index c08d3969734..e18d38cbda8 100644 --- a/src/dune_config_file/dune_config_file.mli +++ b/src/dune_config_file/dune_config_file.mli @@ -2,7 +2,6 @@ module Dune_config : sig (** Dune configuration (visible to the user) *) open Stdune - open Dune_config module Display : module type of Display module Project_defaults : sig @@ -29,6 +28,21 @@ module Dune_config : sig end module Cache : sig + module Toggle : sig + type t = + | Disabled + | Enabled_except_user_rules + | Enabled + + val all : (string * t) list + + val decode + : check:(Dune_lang.Syntax.Version.t -> unit Dune_lang.Decoder.t) + -> t Dune_lang.Decoder.t + + val to_string : t -> string + end + module Storage_mode : sig type t = Dune_cache_storage.Mode.t option @@ -61,7 +75,7 @@ module Dune_config : sig ; concurrency : Concurrency.t field ; terminal_persistence : Terminal_persistence.t field ; sandboxing_preference : Sandboxing_preference.t field - ; cache_enabled : Config.Toggle.t field + ; cache_enabled : Cache.Toggle.t field ; cache_reproducibility_check : Dune_cache.Config.Reproducibility_check.t field ; cache_storage_mode : Cache.Storage_mode.t field ; action_stdout_on_success : Action_output_on_success.t field diff --git a/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t b/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t index 580e2513176..5f6f9ae1d7b 100644 --- a/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t +++ b/test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t @@ -2,7 +2,6 @@ This checks what happens when a file available in the cache is used in a directo $ export DUNE_CACHE_ROOT=$PWD/.cache $ export DUNE_CACHE=enabled - $ export DUNE_CACHE_RULES=enabled $ . ./helpers.sh $ cat > dune-project << EOF diff --git a/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t b/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t index 8d18166a5d2..fc7c7cb3065 100644 --- a/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t +++ b/test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t @@ -1,7 +1,6 @@ We create 2 directory targets which share a whole subdirectory. $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ export DUNE_CACHE=enabled $ . ./helpers.sh diff --git a/test/blackbox-tests/test-cases/directory-targets/cache.t b/test/blackbox-tests/test-cases/directory-targets/cache.t index 5dcbcf06252..dc84fb3a52c 100644 --- a/test/blackbox-tests/test-cases/directory-targets/cache.t +++ b/test/blackbox-tests/test-cases/directory-targets/cache.t @@ -1,7 +1,6 @@ We test that directory targets can go in the shared cache. See #8067. $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ export DUNE_CACHE=enabled In project a, we create a rule with a directory target. The script that creates diff --git a/test/blackbox-tests/test-cases/dune-cache/config.t b/test/blackbox-tests/test-cases/dune-cache/config.t index ad9cce8bb43..b64aae09346 100644 --- a/test/blackbox-tests/test-cases/dune-cache/config.t +++ b/test/blackbox-tests/test-cases/dune-cache/config.t @@ -26,7 +26,6 @@ Check that old cache configuration format works fine with an old language Test that DUNE_CACHE_ROOT can be used to control the cache location $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled Build succeeds and the 'copy' mode is respected diff --git a/test/blackbox-tests/test-cases/dune-cache/dedup.t b/test/blackbox-tests/test-cases/dune-cache/dedup.t index 73b8ef5fdb5..ba9e13e383a 100644 --- a/test/blackbox-tests/test-cases/dune-cache/dedup.t +++ b/test/blackbox-tests/test-cases/dune-cache/dedup.t @@ -2,7 +2,6 @@ Test deduplication of build artifacts when using Dune cache with hard links. $ export DUNE_CACHE=enabled $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ cat > dune-project < (lang dune 2.1) diff --git a/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t b/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t index 01b987c804d..3565ab9d8ea 100644 --- a/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t +++ b/test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t @@ -1,7 +1,6 @@ Check that Dune cache can cope with missing file/metadata entries. $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ cat > config < (lang dune 2.1) diff --git a/test/blackbox-tests/test-cases/dune-cache/mode-copy.t b/test/blackbox-tests/test-cases/dune-cache/mode-copy.t index f0951fa10a5..fedd640c65e 100644 --- a/test/blackbox-tests/test-cases/dune-cache/mode-copy.t +++ b/test/blackbox-tests/test-cases/dune-cache/mode-copy.t @@ -5,7 +5,6 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former. $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime $ export XDG_CACHE_HOME=$PWD/.xdg-cache - $ export DUNE_CACHE_RULES=enabled $ cat > config < (lang dune 3.0) diff --git a/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t b/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t index 11c0f0b6898..07eca4ee470 100644 --- a/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t +++ b/test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t @@ -5,7 +5,6 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former. $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime $ export XDG_CACHE_HOME=$PWD/.xdg-cache - $ export DUNE_CACHE_RULES=enabled $ cat > config < (lang dune 2.1) diff --git a/test/blackbox-tests/test-cases/dune-cache/repro-check.t b/test/blackbox-tests/test-cases/dune-cache/repro-check.t index cbe809b54be..2bcfb0ec0b6 100644 --- a/test/blackbox-tests/test-cases/dune-cache/repro-check.t +++ b/test/blackbox-tests/test-cases/dune-cache/repro-check.t @@ -1,7 +1,6 @@ Test reproducibility check $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ cat > config < (lang dune 3.0) > (cache enabled) diff --git a/test/blackbox-tests/test-cases/dune-cache/size.t/run.t b/test/blackbox-tests/test-cases/dune-cache/size.t/run.t index 46e0c8416c0..a0ec710939b 100644 --- a/test/blackbox-tests/test-cases/dune-cache/size.t/run.t +++ b/test/blackbox-tests/test-cases/dune-cache/size.t/run.t @@ -3,7 +3,6 @@ the cache. $ export DUNE_CACHE=enabled $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ cat > config << EOF > (lang dune 3.7) diff --git a/test/blackbox-tests/test-cases/dune-cache/symlink.t b/test/blackbox-tests/test-cases/dune-cache/symlink.t index d7fcfcb520b..485b46a31b7 100644 --- a/test/blackbox-tests/test-cases/dune-cache/symlink.t +++ b/test/blackbox-tests/test-cases/dune-cache/symlink.t @@ -3,7 +3,6 @@ produced symbolic links work correctly and are appropriately cached. $ export DUNE_CACHE=enabled $ export DUNE_CACHE_ROOT=$PWD/.cache - $ export DUNE_CACHE_RULES=enabled $ cat > dune-project < (lang dune 2.1) diff --git a/test/blackbox-tests/test-cases/dune-cache/trim.t b/test/blackbox-tests/test-cases/dune-cache/trim.t index d4571f0212a..4f397f766c4 100644 --- a/test/blackbox-tests/test-cases/dune-cache/trim.t +++ b/test/blackbox-tests/test-cases/dune-cache/trim.t @@ -1,5 +1,4 @@ $ export DUNE_CACHE=enabled - $ export DUNE_CACHE_RULES=enabled $ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime $ export XDG_CACHE_HOME=$PWD/.xdg-cache diff --git a/test/blackbox-tests/test-cases/pkg/fetch-cache.t b/test/blackbox-tests/test-cases/pkg/fetch-cache.t index 16c4ec61331..3943fffd171 100644 --- a/test/blackbox-tests/test-cases/pkg/fetch-cache.t +++ b/test/blackbox-tests/test-cases/pkg/fetch-cache.t @@ -2,13 +2,12 @@ Testing that files are only fetched once. $ . ./helpers.sh -No need to set DUNE_CACHE (enabled by default) nor DUNE_CACHE_RULES as the +No need to set DUNE_CACHE (enabled by default) as the fetch rules are always considered safe to cache, but we'll set a custom directory for the shared cache. $ export DUNE_CACHE_ROOT=$(pwd)/dune-cache $ unset DUNE_CACHE - $ unset DUNE_CACHE_RULES Set up a project that depends on a package that is being downloaded diff --git a/test/expect-tests/dune_config_file/dune_config_test.ml b/test/expect-tests/dune_config_file/dune_config_test.ml index 4c7ff2e58bc..6f2224340bc 100644 --- a/test/expect-tests/dune_config_file/dune_config_test.ml +++ b/test/expect-tests/dune_config_file/dune_config_test.ml @@ -22,7 +22,7 @@ let%expect_test "cache-check-probability 0.1" = ; concurrency = Fixed 1 ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] - ; cache_enabled = Enabled + ; cache_enabled = Enabled_except_user_rules ; cache_reproducibility_check = Check_with_probability 0.1 ; cache_storage_mode = Some Hardlink ; action_stdout_on_success = Print @@ -45,7 +45,7 @@ let%expect_test "cache-storage-mode copy" = ; concurrency = Fixed 1 ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] - ; cache_enabled = Enabled + ; cache_enabled = Enabled_except_user_rules ; cache_reproducibility_check = Skip ; cache_storage_mode = Some Copy ; action_stdout_on_success = Print @@ -68,7 +68,7 @@ let%expect_test "cache-storage-mode hardlink" = ; concurrency = Fixed 1 ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] - ; cache_enabled = Enabled + ; cache_enabled = Enabled_except_user_rules ; cache_reproducibility_check = Skip ; cache_storage_mode = Some Hardlink ; action_stdout_on_success = Print