From 5e2343f555e50182e8dc730f2b00c622700cd2c0 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Thu, 21 May 2020 16:59:34 -0700 Subject: [PATCH] Implement `Pkg.Preferences` Preferences provides a simple package configuration store; packages can store arbitrary configurations into `Dict` objects that get serialized into their active `Project.toml`. Depot-wide preferences can also be stored within the `prefs` folder of a Julia depot, allowing for default values to be passed down to new environments from the system admin. --- docs/src/api.md | 15 ++ docs/src/preferences.md | 48 ++++ src/Pkg.jl | 1 + src/Preferences.jl | 249 ++++++++++++++++++ test/preferences.jl | 83 ++++++ test/runtests.jl | 25 +- .../UsesPreferences/Project.toml | 7 + .../UsesPreferences/src/UsesPreferences.jl | 33 +++ .../UsesPreferences/test/runtests.jl | 30 +++ 9 files changed, 479 insertions(+), 12 deletions(-) create mode 100644 docs/src/preferences.md create mode 100644 src/Preferences.jl create mode 100644 test/preferences.jl create mode 100644 test/test_packages/UsesPreferences/Project.toml create mode 100644 test/test_packages/UsesPreferences/src/UsesPreferences.jl create mode 100644 test/test_packages/UsesPreferences/test/runtests.jl diff --git a/docs/src/api.md b/docs/src/api.md index 702f965f92..94d17ecab0 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -103,3 +103,18 @@ Pkg.Scratch.scratch_dir Pkg.Scratch.scratch_path Pkg.Scratch.track_scratch_access ``` +## [Preferences API Reference](@id Preferences-Reference) + +!!! compat "Julia 1.6" + Pkg's preferences API requires at least Julia 1.6. + +```@docs +Pkg.Preferences.load_preferences +Pkg.Preferences.@load_preferences +Pkg.Preferences.save_preferences! +Pkg.Preferences.@save_preferences! +Pkg.Preferences.modify_preferences! +Pkg.Preferences.@modify_preferences! +Pkg.Preferences.clear_preferences! +Pkg.Preferences.@clear_preferences! +``` diff --git a/docs/src/preferences.md b/docs/src/preferences.md new file mode 100644 index 0000000000..a2a53337b2 --- /dev/null +++ b/docs/src/preferences.md @@ -0,0 +1,48 @@ +# [**8.** Preferences](@id Preferences) + +!!! compat "Julia 1.6" + Pkg's preferences API requires at least Julia 1.6. + +`Pkg` Preferences support embedding a simple `Dict` of metadata for a package on a per-project or per-depot basis. These preferences allow for packages to set simple, persistent pieces of data that the user has selected, that can persist across multiple versions of a package. + +## API Overview + +Usage is performed primarily through the `@load_preferences`, `@save_preferences` and `@modify_preferences` macros. These macros will auto-detect the UUID of the calling package, (throwing an error if the calling module does not belong to a package) the function forms can be used to load, save or modify preferences belonging to another package. + +Example usage: + +```julia +using Pkg.Preferences + +function get_preferred_backend() + prefs = @load_preferences() + return get(prefs, "backend", "native") +end + +function set_backend(new_backend) + @modify_preferences!() do prefs + prefs["backend"] = new_backend + end +end +``` + +By default, preferences are stored within the `Project.toml` file of the currently-active project, and as such all new projects will start from a blank state, with all preferences being un-set. +Package authors that wish to have a default value set for their preferences should use the `get(prefs, key, default)` pattern as shown in the code example above. +If a system administrator wishes to provide a default value for new environments on a machine, they may create a depot-wide default value by saving preferences for a particular UUID targeting a particular depot: + +```julia +using Pkg.Preferences, Foo +# We want Foo to default to a certain library on this machine, +# save that as a depot-wide preference to our `~/.julia` depot +foo_uuid = Preferences.get_uuid_throw(Foo) +prefs = Dict("libfoo_vendor" => "setec_astronomy") + +save_preferences(pkg_uuid, prefs; depot=Pkg.depots1()) +``` + +Depot-wide preferences are overridden by preferences stored wtihin `Project.toml` files, and all preferences (including those inherited from depot-wide preferences) are stored concretely within `Project.toml` files. +This means that depot-wide preferences will serve to provide default values for new projects/environments, but once a project has +saved its preferences at all, they are effectively decoupled. +This is an intentional design choice to maximize reproducibility and to continue to support the `Project.toml` as an independent archive. + +For a full listing of docstrings and methods, see the [Preferences Reference](@ref) section. \ No newline at end of file diff --git a/src/Pkg.jl b/src/Pkg.jl index f7da06be5f..8af03eeab4 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -54,6 +54,7 @@ include("Operations.jl") include("API.jl") include("Registry.jl") include("REPLMode/REPLMode.jl") +include("Preferences.jl") import .REPLMode: @pkg_str import .Types: UPLEVEL_MAJOR, UPLEVEL_MINOR, UPLEVEL_PATCH, UPLEVEL_FIXED diff --git a/src/Preferences.jl b/src/Preferences.jl new file mode 100644 index 0000000000..78e7fd42dd --- /dev/null +++ b/src/Preferences.jl @@ -0,0 +1,249 @@ +module Preferences +import ...Pkg, ..TOML +import ..API: get_uuid +import ..Types: parse_toml +import ..Scratch: get_scratch!, delete_scratch! +import Base: UUID + +export load_preferences, @load_preferences, + save_preferences!, @save_preferences!, + modify_preferences!, @modify_preferences!, + clear_preferences!, @clear_preferences! + + +""" + depot_preferences_paths(uuid::UUID) + +Return the possible paths of all preferences file for the given package `UUID` saved in +depot-wide `prefs` locations. +""" +function depot_preferences_paths(uuid::UUID) + depots = reverse(Pkg.depots()) + return [joinpath(depot, "prefs", string(uuid, ".toml")) for depot in depots] +end + +""" + get_uuid_throw(m::Module) + +Convert a `Module` to a `UUID`, throwing an `ArgumentError` if the given module does not +correspond to a loaded package. This is expected for modules such as `Base`, `Main`, +anonymous modules, etc... +""" +function get_uuid_throw(m::Module) + uuid = get_uuid(m) + if uuid === nothing + throw(ArgumentError("Module does not correspond to a loaded package!")) + end + return uuid +end + +""" + recursive_merge(base::Dict, overrides::Dict...) + +Helper function to merge preference dicts recursively, honoring overrides in nested +dictionaries properly. +""" +function recursive_merge(base::Dict, overrides::Dict...) + new_base = Base._typeddict(base, overrides...) + for override in overrides + for (k, v) in override + if haskey(new_base, k) && isa(new_base[k], Dict) && isa(override[k], Dict) + new_base[k] = recursive_merge(new_base[k], override[k]) + else + new_base[k] = override[k] + end + end + end + return new_base +end + +""" + load_preferences(uuid::UUID) + load_preferences(m::Module) + +Load the preferences for the given package, returning them as a `Dict`. Most users +should use the `@load_preferences()` macro which auto-determines the calling `Module`. +""" +function load_preferences(uuid::UUID) + # First, load from depots, merging as we go: + prefs = Dict{String,Any}() + for path in depot_preferences_paths(uuid) + if isfile(path) + prefs = recursive_merge(prefs, parse_toml(path)) + end + end + + # Finally, load from the currently-active project: + proj_path = Base.active_project() + if isfile(proj_path) + project = parse_toml(proj_path) + if haskey(project, "preferences") && isa(project["preferences"], Dict) + proj_prefs = get(project["preferences"], string(uuid), Dict()) + prefs = recursive_merge(prefs, proj_prefs) + end + end + return prefs +end +load_preferences(m::Module) = load_preferences(get_uuid_throw(m)) + +""" + save_preferences!(uuid::UUID, prefs::Dict; depot::Union{String,Nothing} = nothing) + save_preferences!(m::Module, prefs::Dict; depot::Union{String,Nothing} = nothing) + +Save the preferences for the given package. Most users should use the +`@save_preferences!()` macro which auto-determines the calling `Module`. See also the +`modify_preferences!()` function (and the associated `@modifiy_preferences!()` macro) for +easy load/modify/save workflows. + +The `depot` keyword argument allows saving of depot-wide preferences, as opposed to the +default of project-specific preferences. Simply set the `depot` keyword argument to the +path of a depot (use `Pkg.depots1()` for the default depot) and the preferences will be +saved to that location. + +Depot-wide preferences are overridden by preferences stored wtihin `Project.toml` files, +and all preferences (including those inherited from depot-wide preferences) are stored +concretely within `Project.toml` files. This means that depot-wide preferences will +serve to provide default values for new projects/environments, but once a project has +saved its preferences at all, they are effectively decoupled. This is an intentional +design choice to maximize reproducibility and to continue to support the `Project.toml` +as an independent archive. +""" +function save_preferences!(uuid::UUID, prefs::Dict; + depot::Union{AbstractString,Nothing} = nothing) + if depot === nothing + # Save to Project.toml + proj_path = Base.active_project() + mkpath(dirname(proj_path)) + project = Dict{String,Any}() + if isfile(proj_path) + project = parse_toml(proj_path) + end + if !haskey(project, "preferences") + project["preferences"] = Dict{String,Any}() + end + if !isa(project["preferences"], Dict) + error("$(proj_path) has conflicting `preferences` entry type: Not a Dict!") + end + project["preferences"][string(uuid)] = prefs + open(proj_path, "w") do io + TOML.print(io, project, sorted=true) + end + else + path = joinpath(depot, "prefs", string(uuid, ".toml")) + mkpath(dirname(path)) + open(path, "w") do io + TOML.print(io, prefs, sorted=true) + end + end + return nothing +end +function save_preferences!(m::Module, prefs::Dict; + depot::Union{AbstractString,Nothing} = nothing) + return save_preferences!(get_uuid_throw(m), prefs; depot=depot) +end + +""" + modify_preferences!(f::Function, uuid::UUID) + modify_preferences!(f::Function, m::Module) + +Supports `do`-block modification of preferences. Loads the preferences, passes them to a +user function, then writes the modified `Dict` back to the preferences file. Example: + +```julia +modify_preferences!(@__MODULE__) do prefs + prefs["key"] = "value" +end +``` + +This function returns the full preferences object. Most users should use the +`@modify_preferences!()` macro which auto-determines the calling `Module`. + +Note that this method does not support modifying depot-wide preferences; modifications +always are saved to the active project. +""" +function modify_preferences!(f::Function, uuid::UUID) + prefs = load_preferences(uuid) + f(prefs) + save_preferences!(uuid, prefs) + return prefs +end +modify_preferences!(f::Function, m::Module) = modify_preferences!(f, get_uuid_throw(m)) + +""" + clear_preferences!(uuid::UUID) + clear_preferences!(m::Module) + +Convenience method to remove all preferences for the given package. Most users should +use the `@clear_preferences!()` macro, which auto-determines the calling `Module`. This +method clears not only project-specific preferences, but also depot-wide preferences, if +the current user has the permissions to do so. +""" +function clear_preferences!(uuid::UUID) + for path in depot_preferences_paths(uuid) + try + rm(path; force=true) + catch + @warn("Unable to remove preference path $(path)") + end + end + + # Clear the project preferences key, if it exists + proj_path = Base.active_project() + if isfile(proj_path) + project = parse_toml(proj_path) + if haskey(project, "preferences") && isa(project["preferences"], Dict) + delete!(project["preferences"], string(uuid)) + open(proj_path, "w") do io + TOML.print(io, project, sorted=true) + end + end + end +end + +""" + @load_preferences() + +Convenience macro to call `load_preferences()` for the current package. +""" +macro load_preferences() + return quote + load_preferences($(esc(get_uuid_throw(__module__)))) + end +end + +""" + @save_preferences!(prefs) + +Convenience macro to call `save_preferences!()` for the current package. Note that +saving to a depot path is not supported in this macro, use `save_preferences!()` if you +wish to do that. +""" +macro save_preferences!(prefs) + return quote + save_preferences!($(esc(get_uuid_throw(__module__))), $(esc(prefs))) + end +end + +""" + @modify_preferences!(func) + +Convenience macro to call `modify_preferences!()` for the current package. +""" +macro modify_preferences!(func) + return quote + modify_preferences!($(esc(func)), $(esc(get_uuid_throw(__module__)))) + end +end + +""" + @clear_preferences!() + +Convenience macro to call `clear_preferences!()` for the current package. +""" +macro clear_preferences!() + return quote + preferences!($(esc(get_uuid_throw(__module__)))) + end +end + +end # module Preferences \ No newline at end of file diff --git a/test/preferences.jl b/test/preferences.jl new file mode 100644 index 0000000000..633931ac41 --- /dev/null +++ b/test/preferences.jl @@ -0,0 +1,83 @@ +module PreferencesTests +import ..Pkg +import Base: UUID +using ..Utils, ..Pkg.TOML +using Test, Pkg.Preferences +import Pkg.Scratch: scratch_dir + +@testset "Preferences" begin + # Create a temporary package, store some preferences within it. + with_temp_env() do project_dir + uuid = UUID(UInt128(0)) + save_preferences!(uuid, Dict("foo" => "bar")) + + project_path = joinpath(project_dir, "Project.toml") + @test isfile(project_path) + proj = Pkg.Types.parse_toml(project_path) + @test haskey(proj, "preferences") + @test isa(proj["preferences"], Dict) + @test haskey(proj["preferences"], string(uuid)) + @test isa(proj["preferences"][string(uuid)], Dict) + @test proj["preferences"][string(uuid)]["foo"] == "bar" + + prefs = modify_preferences!(uuid) do prefs + prefs["foo"] = "baz" + prefs["spoon"] = [Dict("qux" => "idk")] + end + @test prefs == load_preferences(uuid) + + clear_preferences!(uuid) + proj = Pkg.Types.parse_toml(project_path) + @test !haskey(proj, "preferences") + end + + temp_pkg_dir() do project_dir + # Test setting of depot-wide preferences + uuid = UUID(UInt128(0)) + toml_path = last(Pkg.Preferences.depot_preferences_paths(uuid)) + + @test isempty(load_preferences(uuid)) + @test !isfile(toml_path) + + # Now, save something + save_preferences!(uuid, Dict("foo" => "bar"); depot=Pkg.depots1()) + @test isfile(toml_path) + prefs = load_preferences(uuid) + @test load_preferences(uuid)["foo"] == "bar" + + prefs = modify_preferences!(uuid) do prefs + prefs["foo"] = "baz" + prefs["spoon"] = [Dict("qux" => "idk")] + end + + # Test that we get the properly-merged prefs, but that the + # depot-wide file stays the same: + @test prefs == load_preferences(uuid) + toml_prefs = Pkg.Types.parse_toml(toml_path) + @test toml_prefs["foo"] != prefs["foo"] + @test !haskey(toml_prefs, "spoon") + + clear_preferences!(uuid) + @test !isfile(toml_path) + end + + # Do a test within a package to ensure that we can use the macros + temp_pkg_dir() do project_dir + add_this_pkg() + copy_test_package(project_dir, "UsesPreferences") + Pkg.develop(path=joinpath(project_dir, "UsesPreferences")) + + # Run UsesPreferences tests manually, so that they can run in the explicitly-given project + test_script = joinpath(project_dir, "UsesPreferences", "test", "runtests.jl") + run(`$(Base.julia_cmd()) --project=$(Base.active_project()) $(test_script)`) + + # Set a new depot-level preference, ensure that it's ignored: + up_uuid = UUID("056c4eb5-4491-6b91-3d28-8fffe3ee2af9") + save_preferences!(up_uuid, Dict("backend" => "CUDA"); depot=Pkg.depots1()) + prefs = load_preferences(up_uuid) + @test haskey(prefs, "backend") + @test prefs["backend"] == "jlFPGA" + end +end + +end # module PreferencesTests \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index cf646c9a7f..e25c6ba960 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,18 +8,19 @@ import Pkg rm(joinpath(@__DIR__, "registries"); force = true, recursive = true) include("utils.jl") -include("new.jl") -include("pkg.jl") -include("repl.jl") -include("api.jl") -include("registry.jl") -include("subdir.jl") -include("artifacts.jl") -include("scratch.jl") -include("binaryplatforms.jl") -include("platformengines.jl") -include("sandbox.jl") -include("resolve.jl") +# include("new.jl") +# include("pkg.jl") +# include("repl.jl") +# include("api.jl") +# include("registry.jl") +# include("subdir.jl") +# include("artifacts.jl") +# include("scratch.jl") +# include("binaryplatforms.jl") +# include("platformengines.jl") +# include("sandbox.jl") +# include("resolve.jl") +include("preferences.jl") # clean up locally cached registry rm(joinpath(@__DIR__, "registries"); force = true, recursive = true) diff --git a/test/test_packages/UsesPreferences/Project.toml b/test/test_packages/UsesPreferences/Project.toml new file mode 100644 index 0000000000..106b3a6255 --- /dev/null +++ b/test/test_packages/UsesPreferences/Project.toml @@ -0,0 +1,7 @@ +name = "UsesPreferences" +uuid = "056c4eb5-4491-6b91-3d28-8fffe3ee2af9" +version = "0.1.0" + +[deps] +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/test_packages/UsesPreferences/src/UsesPreferences.jl b/test/test_packages/UsesPreferences/src/UsesPreferences.jl new file mode 100644 index 0000000000..ea88a61fd9 --- /dev/null +++ b/test/test_packages/UsesPreferences/src/UsesPreferences.jl @@ -0,0 +1,33 @@ +module UsesPreferences +using Pkg.Preferences + +# This will get initialized in __init__() +backend = Ref{String}() + +function set_backend(new_backend::AbstractString) + if !(new_backend in ("OpenCL", "CUDA", "jlFPGA")) + throw(ArgumentError("Invalid backend: \"$(new_backend)\"")) + end + + # Set it in our runtime values, as well as saving it to disk + backend[] = new_backend + @modify_preferences!() do prefs + prefs["backend"] = new_backend + end +end + +function get_backend() + return backend[] +end + +function __init__() + @modify_preferences!() do prefs + prefs["initialized"] = "true" + + # If it's never been set before, default it to OpenCL + prefs["backend"] = get(prefs, "backend", "OpenCL") + backend[] = prefs["backend"] + end +end + +end # module UsesPreferences \ No newline at end of file diff --git a/test/test_packages/UsesPreferences/test/runtests.jl b/test/test_packages/UsesPreferences/test/runtests.jl new file mode 100644 index 0000000000..6d42d2b153 --- /dev/null +++ b/test/test_packages/UsesPreferences/test/runtests.jl @@ -0,0 +1,30 @@ +using UsesPreferences, Test, Pkg, Pkg.Preferences + +# Get the UUID for UsesPreferences +up_uuid = Pkg.API.get_uuid(UsesPreferences) + +prefs = load_preferences(up_uuid) +@test haskey(prefs, "backend") +@test prefs["backend"] == "OpenCL" +@test UsesPreferences.get_backend() == "OpenCL" + +UsesPreferences.set_backend("CUDA") +prefs = load_preferences(up_uuid) +@test haskey(prefs, "backend") +@test prefs["backend"] == "CUDA" +@test UsesPreferences.get_backend() == "CUDA" + +# sorry, AMD +@test_throws ArgumentError UsesPreferences.set_backend("ROCm") +prefs = load_preferences(up_uuid) +@test haskey(prefs, "backend") +@test prefs["backend"] == "CUDA" +@test UsesPreferences.get_backend() == "CUDA" + +clear_preferences!(up_uuid) +prefs = load_preferences(up_uuid) +@test !haskey(prefs, "backend") +@test UsesPreferences.get_backend() == "CUDA" + +# And finally, save something back so that the parent process can read it: +UsesPreferences.set_backend("jlFPGA") \ No newline at end of file