diff --git a/base/loading.jl b/base/loading.jl index 776f06202e0b4..da4e21ed7d25b 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -163,16 +163,73 @@ function version_slug(uuid::UUID, sha1::SHA1, p::Int=5) return slug(crc, p) end -struct TOMLCache - p::TOML.Parser - d::Dict{String, Dict{String, Any}} +mutable struct CachedTOMLDict + path::String + inode::UInt64 + mtime::Float64 + size::Int64 + hash::UInt32 + d::Dict{String, Any} +end + +function CachedTOMLDict(p::TOML.Parser, path::String) + s = stat(path) + content = read(path) + crc32 = _crc32c(content) + TOML.reinit!(p, String(content); filepath=path) + d = TOML.parse(p) + return CachedTOMLDict( + path, + s.inode, + s.mtime, + s.size, + crc32, + d, + ) +end + +function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict) + s = stat(f.path) + time_since_cached = time() - f.mtime + rough_mtime_granularity = 0.1 # seconds + # In case the file is being updated faster than the mtime granularity, + # and have the same size after the update we might miss that it changed. Therefore + # always check the hash in case we recently created the cache. + if time_since_cached < rough_mtime_granularity || s.inode != f.inode || s.mtime != f.mtime || f.size != s.size + content = read(f.path) + new_hash = _crc32c(content) + if new_hash != f.hash + f.inode = s.inode + f.mtime = s.mtime + f.size = s.size + f.hash = new_hash + @debug "Cache of TOML file $(repr(f.path)) invalid, reparsing..." + TOML.reinit!(p, String(content); filepath=f.path) + return f.d = TOML.parse(p) + end + end + return f.d end -TOMLCache() = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) -function parsed_toml(cache::TOMLCache, project_file::String) - get!(cache.d, project_file) do - TOML.reinit!(cache.p, read(project_file, String); filepath=project_file) - TOML.parse(cache.p) +struct TOMLCache + p::TOML.Parser + d::Dict{String, CachedTOMLDict} +end +const TOML_CACHE = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) + +const TOML_LOCK = ReentrantLock() +parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, TOML_LOCK) +function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock) + lock(toml_lock) do + if !haskey(toml_cache.d, project_file) + @debug "Creating new cache for $(repr(project_file))" + d = CachedTOMLDict(toml_cache.p, project_file) + toml_cache.d[project_file] = d + return d.d + else + d = toml_cache.d[project_file] + return get_updated_dict(toml_cache.p, d) + end end end @@ -180,22 +237,21 @@ end # Used by Pkg but not used in loading itself function find_package(arg) - cache = TOMLCache() - pkg = identify_package(arg, cache) + pkg = identify_package(arg) pkg === nothing && return nothing - return locate_package(pkg, cache) + return locate_package(pkg) end ## package identity: given a package name and a context, try to return its identity ## -identify_package(where::Module, name::String, cache::TOMLCache = TOMLCache()) = identify_package(PkgId(where), name, cache) +identify_package(where::Module, name::String) = identify_package(PkgId(where), name) # identify_package computes the PkgId for `name` from the context of `where` # or return `nothing` if no mapping exists for it -function identify_package(where::PkgId, name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} +function identify_package(where::PkgId, name::String)::Union{Nothing,PkgId} where.name === name && return where - where.uuid === nothing && return identify_package(name, cache) # ignore `where` + where.uuid === nothing && return identify_package(name) # ignore `where` for env in load_path() - uuid = manifest_deps_get(env, where, name, cache) + uuid = manifest_deps_get(env, where, name) uuid === nothing && continue # not found--keep looking uuid.uuid === nothing || return uuid # found in explicit environment--use it return nothing # found in implicit environment--return "not found" @@ -205,33 +261,33 @@ end # identify_package computes the PkgId for `name` from toplevel context # by looking through the Project.toml files and directories -function identify_package(name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} +function identify_package(name::String)::Union{Nothing,PkgId} for env in load_path() - uuid = project_deps_get(env, name, cache) + uuid = project_deps_get(env, name) uuid === nothing || return uuid # found--return it end return nothing end ## package location: given a package identity, find file to load ## -function locate_package(pkg::PkgId, cache::TOMLCache=TOMLCache())::Union{Nothing,String} +function locate_package(pkg::PkgId)::Union{Nothing,String} if pkg.uuid === nothing for env in load_path() # look for the toplevel pkg `pkg.name` in this entry - found = project_deps_get(env, pkg.name, cache) + found = project_deps_get(env, pkg.name) found === nothing && continue if pkg == found # pkg.name is present in this directory or project file, # return the path the entry point for the code, if it could be found # otherwise, signal failure - return implicit_manifest_uuid_path(env, pkg, cache) + return implicit_manifest_uuid_path(env, pkg) end @assert found.uuid !== nothing - return locate_package(found, cache) # restart search now that we know the uuid for pkg + return locate_package(found) # restart search now that we know the uuid for pkg end else for env in load_path() - path = manifest_uuid_path(env, pkg, cache) + path = manifest_uuid_path(env, pkg) path === nothing || return entry_path(path, pkg.name) end end @@ -291,50 +347,50 @@ function env_project_file(env::String)::Union{Bool,String} return false end -function project_deps_get(env::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} +function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} project_file = env_project_file(env) if project_file isa String - pkg_uuid = explicit_project_deps_get(project_file, name, cache) + pkg_uuid = explicit_project_deps_get(project_file, name) pkg_uuid === nothing || return PkgId(pkg_uuid, name) elseif project_file - return implicit_project_deps_get(env, name, cache) + return implicit_project_deps_get(env, name) end return nothing end -function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} +function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = env_project_file(env) if project_file isa String # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name, cache) + proj = project_file_name_uuid(project_file, where.name) if proj == where # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name, cache) + pkg_uuid = explicit_project_deps_get(project_file, name) return PkgId(pkg_uuid, name) end # look for manifest file and `where` stanza - return explicit_manifest_deps_get(project_file, where.uuid, name, cache) + return explicit_manifest_deps_get(project_file, where.uuid, name) elseif project_file # if env names a directory, search it - return implicit_manifest_deps_get(env, where, name, cache) + return implicit_manifest_deps_get(env, where, name) end return nothing end -function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache) +function uuid_in_environment(project_file::String, uuid::UUID) # First, check to see if we're looking for the environment itself - proj_uuid = get(parsed_toml(cache, project_file), "uuid", nothing) + proj_uuid = get(parsed_toml(project_file), "uuid", nothing) if proj_uuid !== nothing && UUID(proj_uuid) == uuid return true end # Check to see if there's a Manifest.toml associated with this project - manifest_file = project_file_manifest_path(project_file, cache) + manifest_file = project_file_manifest_path(project_file) if manifest_file === nothing return false end - manifest = parsed_toml(cache, manifest_file) + manifest = parsed_toml(manifest_file) for (dep_name, entries) in manifest for entry in entries entry_uuid = get(entry, "uuid", nothing)::Union{String, Nothing} @@ -347,42 +403,42 @@ function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache) return false end -function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} +function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String} project_file = env_project_file(env) if project_file isa String - proj = project_file_name_uuid(project_file, pkg.name, cache) + proj = project_file_name_uuid(project_file, pkg.name) if proj == pkg # if `pkg` matches the project, return the project itself - return project_file_path(project_file, pkg.name, cache) + return project_file_path(project_file, pkg.name) end # look for manifest file and `where` stanza - return explicit_manifest_uuid_path(project_file, pkg, cache) + return explicit_manifest_uuid_path(project_file, pkg) elseif project_file # if env names a directory, search it - return implicit_manifest_uuid_path(env, pkg, cache) + return implicit_manifest_uuid_path(env, pkg) end return nothing end # find project file's top-level UUID entry (or nothing) -function project_file_name_uuid(project_file::String, name::String, cache::TOMLCache)::PkgId +function project_file_name_uuid(project_file::String, name::String)::PkgId uuid = dummy_uuid(project_file) - d = parsed_toml(cache, project_file) + d = parsed_toml(project_file) uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} uuid′ === nothing || (uuid = UUID(uuid′)) name = get(d, "name", name)::String return PkgId(uuid, name) end -function project_file_path(project_file::String, name::String, cache) - d = parsed_toml(cache, project_file) +function project_file_path(project_file::String, name::String) + d = parsed_toml(project_file) joinpath(dirname(project_file), get(d, "path", "")::String) end # find project file's corresponding manifest file -function project_file_manifest_path(project_file::String, cache::TOMLCache)::Union{Nothing,String} +function project_file_manifest_path(project_file::String)::Union{Nothing,String} dir = abspath(dirname(project_file)) - d = parsed_toml(cache, project_file) + d = parsed_toml(project_file) explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} if explicit_manifest !== nothing manifest_file = normpath(joinpath(dir, explicit_manifest)) @@ -434,8 +490,8 @@ end # find project file root or deps `name => uuid` mapping # return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String, cache::TOMLCache)::Union{Nothing,UUID} - d = parsed_toml(cache, project_file) +function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} + d = parsed_toml(project_file) root_uuid = dummy_uuid(project_file) if get(d, "name", nothing)::Union{String, Nothing} === name uuid = get(d, "uuid", nothing)::Union{String, Nothing} @@ -451,10 +507,10 @@ end # find `where` stanza and return the PkgId for `name` # return `nothing` if it did not find `where` (indicating caller should continue searching) -function explicit_manifest_deps_get(project_file::String, where::UUID, name::String, cache::TOMLCache)::Union{Nothing,PkgId} - manifest_file = project_file_manifest_path(project_file, cache) +function explicit_manifest_deps_get(project_file::String, where::UUID, name::String)::Union{Nothing,PkgId} + manifest_file = project_file_manifest_path(project_file) manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH - d = parsed_toml(cache, manifest_file) + d = parsed_toml(manifest_file) found_where = false found_name = false for (dep_name, entries) in d @@ -498,11 +554,11 @@ function explicit_manifest_deps_get(project_file::String, where::UUID, name::Str end # find `uuid` stanza, return the corresponding path -function explicit_manifest_uuid_path(project_file::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} - manifest_file = project_file_manifest_path(project_file, cache) +function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String} + manifest_file = project_file_manifest_path(project_file) manifest_file === nothing && return nothing # no manifest, skip env - d = parsed_toml(cache, manifest_file) + d = parsed_toml(manifest_file) entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}} entries === nothing && return nothing # TODO: allow name to mismatch? for entry in entries @@ -539,13 +595,13 @@ end # look for an entry point for `name` from a top-level package (no environment) # otherwise return `nothing` to indicate the caller should keep searching -function implicit_project_deps_get(dir::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} +function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} path, project_file = entry_point_and_project_file(dir, name) if project_file === nothing path === nothing && return nothing return PkgId(name) end - proj = project_file_name_uuid(project_file, name, cache) + proj = project_file_name_uuid(project_file, name) proj.name == name || return nothing return proj end @@ -553,25 +609,25 @@ end # look for an entry-point for `name`, check that UUID matches # if there's a project file, look up `name` in its deps and return that # otherwise return `nothing` to indicate the caller should keep searching -function implicit_manifest_deps_get(dir::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} +function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = entry_point_and_project_file(dir, where.name)[2] project_file === nothing && return nothing # a project file is mandatory for a package with a uuid - proj = project_file_name_uuid(project_file, where.name, cache) + proj = project_file_name_uuid(project_file, where.name, ) proj == where || return nothing # verify that this is the correct project file # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name, cache) + pkg_uuid = explicit_project_deps_get(project_file, name) return PkgId(pkg_uuid, name) end # look for an entry-point for `pkg` and return its path if UUID matches -function implicit_manifest_uuid_path(dir::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} +function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} path, project_file = entry_point_and_project_file(dir, pkg.name) if project_file === nothing pkg.uuid === nothing || return nothing return path end - proj = project_file_name_uuid(project_file, pkg.name, cache) + proj = project_file_name_uuid(project_file, pkg.name) proj == pkg || return nothing return path end @@ -631,7 +687,7 @@ function _include_from_serialized(path::String, depmods::Vector{Any}) return restored end -function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}, cache::TOMLCache) +function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}) if root_module_exists(modkey) M = root_module(modkey) if PkgId(M) == modkey && module_build_id(M) === build_id @@ -639,10 +695,10 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U end else if modpath === nothing - modpath = locate_package(modkey, cache) + modpath = locate_package(modkey) modpath === nothing && return nothing end - mod = _require_search_from_serialized(modkey, String(modpath), cache) + mod = _require_search_from_serialized(modkey, String(modpath)) get!(PkgOrigin, pkgorigins, modkey).path = modpath if !isa(mod, Bool) for callback in package_callbacks @@ -659,7 +715,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U return nothing end -function _require_from_serialized(path::String, cache::TOMLCache) +function _require_from_serialized(path::String) # loads a precompile cache file, ignoring stale_cachfile tests # load all of the dependent modules first local depmodnames @@ -675,7 +731,7 @@ function _require_from_serialized(path::String, cache::TOMLCache) depmods = Vector{Any}(undef, ndeps) for i in 1:ndeps modkey, build_id = depmodnames[i] - dep = _tryrequire_from_serialized(modkey, build_id, nothing, cache) + dep = _tryrequire_from_serialized(modkey, build_id, nothing) dep === nothing && return ErrorException("Required dependency $modkey failed to load from a cache file.") depmods[i] = dep::Module end @@ -686,10 +742,10 @@ end # returns `true` if require found a precompile cache for this sourcepath, but couldn't load it # returns `false` if the module isn't known to be precompilable # returns the set of modules restored if the cache load succeeded -function _require_search_from_serialized(pkg::PkgId, sourcepath::String, cache::TOMLCache) +function _require_search_from_serialized(pkg::PkgId, sourcepath::String) paths = find_all_in_cache_path(pkg) for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(sourcepath, path_to_try, cache) + staledeps = stale_cachefile(sourcepath, path_to_try) if staledeps === true continue end @@ -702,7 +758,7 @@ function _require_search_from_serialized(pkg::PkgId, sourcepath::String, cache:: dep = staledeps[i] dep isa Module && continue modpath, modkey, build_id = dep::Tuple{String, PkgId, UInt64} - dep = _tryrequire_from_serialized(modkey, build_id, modpath, cache) + dep = _tryrequire_from_serialized(modkey, build_id, modpath) if dep === nothing @debug "Required dependency $modkey failed to load from cache file for $modpath." staledeps = true @@ -819,8 +875,7 @@ For more details regarding code loading, see the manual sections on [modules](@r [parallel computing](@ref code-availability). """ function require(into::Module, mod::Symbol) - cache = TOMLCache() - uuidkey = identify_package(into, String(mod), cache) + uuidkey = identify_package(into, String(mod)) # Core.println("require($(PkgId(into)), $mod) -> $uuidkey") if uuidkey === nothing where = PkgId(into) @@ -855,7 +910,7 @@ function require(into::Module, mod::Symbol) if _track_dependencies[] push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end - return require(uuidkey, cache) + return require(uuidkey) end mutable struct PkgOrigin @@ -866,9 +921,9 @@ end PkgOrigin() = PkgOrigin(nothing, nothing) const pkgorigins = Dict{PkgId,PkgOrigin}() -function require(uuidkey::PkgId, cache::TOMLCache=TOMLCache()) +function require(uuidkey::PkgId) if !root_module_exists(uuidkey) - cachefile = _require(uuidkey, cache) + cachefile = _require(uuidkey) if cachefile !== nothing get!(PkgOrigin, pkgorigins, uuidkey).cachepath = cachefile end @@ -927,7 +982,7 @@ function unreference_module(key::PkgId) end # Returns `nothing` or the name of the newly-created cachefile -function _require(pkg::PkgId, cache::TOMLCache) +function _require(pkg::PkgId) # handle recursive calls to require loading = get(package_locks, pkg, false) if loading !== false @@ -941,7 +996,7 @@ function _require(pkg::PkgId, cache::TOMLCache) try toplevel_load[] = false # perform the search operation to select the module file require intends to load - path = locate_package(pkg, cache) + path = locate_package(pkg) get!(PkgOrigin, pkgorigins, pkg).path = path if path === nothing throw(ArgumentError(""" @@ -952,7 +1007,7 @@ function _require(pkg::PkgId, cache::TOMLCache) # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 - m = _require_search_from_serialized(pkg, path, cache) + m = _require_search_from_serialized(pkg, path) if !isa(m, Bool) return end @@ -975,7 +1030,7 @@ function _require(pkg::PkgId, cache::TOMLCache) if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0) # spawn off a new incremental pre-compile task for recursive `require` calls # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) - cachefile = compilecache(pkg, path, cache) + cachefile = compilecache(pkg, path) if isa(cachefile, Exception) if precompilableerror(cachefile) verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug @@ -985,7 +1040,7 @@ function _require(pkg::PkgId, cache::TOMLCache) end # fall-through to loading the file locally else - m = _require_from_serialized(cachefile, cache) + m = _require_from_serialized(cachefile) if isa(m, Exception) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else @@ -1149,10 +1204,12 @@ function load_path_setup_code(load_path::Bool=true) end # this is called in the external process that generates precompiled package files -function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) +function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, + concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}, toml_cache::Dict{String, CachedTOMLDict}) append!(empty!(Base.DEPOT_PATH), depot_path) append!(empty!(Base.DL_LOAD_PATH), dl_load_path) append!(empty!(Base.LOAD_PATH), load_path) + copy!(Base.TOML_CACHE.d, toml_cache) ENV["JULIA_LOAD_PATH"] = join(load_path, Sys.iswindows() ? ';' : ':') Base.ACTIVE_PROJECT[] = nothing Base._track_dependencies[] = true @@ -1174,8 +1231,8 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end end -@assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),Nothing)) -@assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),String)) +@assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),Nothing, Dict{String,CachedTOMLDict})) +@assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),String, Dict{String,CachedTOMLDict})) const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), show_errors::Bool = true) @@ -1211,7 +1268,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d # write data over stdin to avoid the (unlikely) case of exceeding max command line size write(io.in, """ Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), - $(repr(load_path)), $deps, $(repr(source_path(nothing)))) + $(repr(load_path)), $deps, $(repr(source_path(nothing))), $(repr(TOML_CACHE.d))) """) close(io.in) return io @@ -1220,7 +1277,7 @@ end @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool)) @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool)) -function compilecache_path(pkg::PkgId, cache::TOMLCache)::String +function compilecache_path(pkg::PkgId)::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) isdir(cachepath) || mkpath(cachepath) @@ -1230,7 +1287,7 @@ function compilecache_path(pkg::PkgId, cache::TOMLCache)::String crc = _crc32c(something(Base.active_project(), "")) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) - crc = _crc32c(get_preferences_hash(pkg.uuid, cache), crc) + crc = _crc32c(get_preferences_hash(pkg.uuid), crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) end @@ -1244,17 +1301,17 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache(), show_errors::Bool = true) - path = locate_package(pkg, cache) +function compilecache(pkg::PkgId, show_errors::Bool = true) + path = locate_package(pkg) path === nothing && throw(ArgumentError("$pkg not found during precompilation")) - return compilecache(pkg, path, cache, show_errors) + return compilecache(pkg, path, show_errors) end const MAX_NUM_PRECOMPILE_FILES = 10 -function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(), show_errors::Bool = true) +function compilecache(pkg::PkgId, path::String, show_errors::Bool = true, do_logging::Bool=true) # decide where to put the resulting cache file - cachefile = compilecache_path(pkg, cache) + cachefile = compilecache_path(pkg) cachepath = dirname(cachefile) # prune the directory with cache files if pkg.uuid !== nothing @@ -1265,7 +1322,7 @@ function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(), rm(joinpath(cachepath, cachefiles[idx])) end end - # build up the list of modules that we want the precompile process to preserve + # build up the list of modules that we want the` precompile process to preserve concrete_deps = copy(_concrete_dependencies) for (key, mod) in loaded_modules if !(mod === Main || mod === Core || mod === Base) @@ -1456,24 +1513,24 @@ function srctext_files(f::IO, srctextpos::Int64) end # Find the Project.toml that we should load/store to for Preferences -function get_preferences_project_path(uuid::UUID, cache::TOMLCache = TOMLCache()) +function get_preferences_project_path(uuid::UUID) for env in load_path() project_file = env_project_file(env) if !isa(project_file, String) continue end - if uuid_in_environment(project_file, uuid, cache) + if uuid_in_environment(project_file, uuid) return project_file end end return nothing end -function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache(); +function get_preferences(uuid::UUID; prefs_key::String = "compile-preferences") - project_path = get_preferences_project_path(uuid, cache) + project_path = get_preferences_project_path(uuid) if project_path !== nothing - preferences = get(parsed_toml(cache, project_path), prefs_key, Dict{String,Any}()) + preferences = get(parsed_toml(project_path), prefs_key, Dict{String,Any}()) if haskey(preferences, string(uuid)) return preferences[string(uuid)] end @@ -1481,15 +1538,14 @@ function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache(); # Fall back to default value of "no preferences". return Dict{String,Any}() end -get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache()) = UInt64(hash(get_preferences(uuid, cache))) -get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache) -get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}())) +get_preferences_hash(uuid::UUID) = UInt64(hash(get_preferences(uuid))) +get_preferences_hash(m::Module) = get_preferences_hash(PkgId(m).uuid) +get_preferences_hash(::Nothing) = UInt64(hash(Dict{String,Any}())) # returns true if it "cachefile.ji" is stale relative to "modpath.jl" # otherwise returns the list of dependencies to also check -stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache()) -function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) +function stale_cachefile(modpath::String, cachefile::String) io = open(cachefile, "r") try if !isvalid_cache_header(io) @@ -1515,7 +1571,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) return true # Won't be able to fulfill dependency end else - path = locate_package(req_key, cache) + path = locate_package(req_key) get!(PkgOrigin, pkgorigins, req_key).path = path if path === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." @@ -1549,7 +1605,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) end for (modkey, req_modkey) in requires # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` - if identify_package(modkey, req_modkey.name, cache) != req_modkey + if identify_package(modkey, req_modkey.name) != req_modkey @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" return true end @@ -1572,7 +1628,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) end if isa(id, PkgId) - curr_prefs_hash = get_preferences_hash(id.uuid, cache) + curr_prefs_hash = get_preferences_hash(id.uuid) if prefs_hash != curr_prefs_hash @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" return true diff --git a/base/sysimg.jl b/base/sysimg.jl index d114a16be8c71..60987eb904a80 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -110,6 +110,8 @@ let empty!(DEPOT_PATH) end +empty!(Base.TOML_CACHE.d) +Base.TOML.reinit!(Base.TOML_CACHE.p, "") @eval Sys begin BINDIR = "" STDLIB = "" diff --git a/base/toml_parser.jl b/base/toml_parser.jl index d943ce9f6ae89..9bd31905e9fb8 100644 --- a/base/toml_parser.jl +++ b/base/toml_parser.jl @@ -102,7 +102,7 @@ function Parser(str::String; filepath=nothing) IdSet{TOMLDict}(), # defined_tables root, filepath, - get(Base.loaded_modules, DATES_PKGID, nothing), + isdefined(Base, :loaded_modules) ? get(Base.loaded_modules, DATES_PKGID, nothing) : nothing, ) startup(l) return l diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index a0047f0f7767e..29c28fc84f3d6 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String}) for proj in Base.project_names project_file = joinpath(root, proj) if Base.isfile_casesensitive(project_file) - pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache()) + pkgid = Base.project_file_name_uuid(project_file, "") isempty(pkgid.name) && return path # bad Project file # return the joined the module name prefix and path suffix path = path[nextind(path, sizeof(root)):end] diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index e51b811ed9d6f..1bd04639b0f47 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -583,15 +583,16 @@ end function project_deps_get_completion_candidates(pkgstarts::String, project_file::String) loading_candidates = String[] - p = Base.TOML.Parser() - Base.TOML.reinit!(p, read(project_file, String); filepath=project_file) - d = Base.TOML.parse(p) - pkg = get(d, "name", nothing) + d = Base.parsed_toml(project_file) + pkg = get(d, "name", nothing)::Union{String, Nothing} if pkg !== nothing && startswith(pkg, pkgstarts) push!(loading_candidates, pkg) end - for (pkg, _) in get(d, "deps", []) - startswith(pkg, pkgstarts) && push!(loading_candidates, pkg) + deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing} + if deps !== nothing + for (pkg, _) in deps + startswith(pkg, pkgstarts) && push!(loading_candidates, pkg) + end end return Completion[PackageCompletion(name) for name in loading_candidates] end diff --git a/test/loading.jl b/test/loading.jl index 637382d49f8b1..89bc6887e9b64 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -176,10 +176,9 @@ end d = findfirst(line -> line == "[deps]", p) t = findfirst(line -> startswith(line, "This"), p) # look up various packages by name - cache = Base.TOMLCache() - root = Base.explicit_project_deps_get(project_file, "Root", cache) - this = Base.explicit_project_deps_get(project_file, "This", cache) - that = Base.explicit_project_deps_get(project_file, "That", cache) + root = Base.explicit_project_deps_get(project_file, "Root") + this = Base.explicit_project_deps_get(project_file, "This") + that = Base.explicit_project_deps_get(project_file, "That") # test that the correct answers are given @test root == (something(n, N+1) ≥ something(d, N+1) ? nothing : something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid) @@ -205,13 +204,13 @@ Base.ACTIVE_PROJECT[] = nothing # locate `tail(names)` package by following the search path graph through `names` starting from `where` function recurse_package(where::PkgId, name::String, names::String...) - pkg = identify_package(where, name, Base.TOMLCache()) + pkg = identify_package(where, name) pkg === nothing && return nothing return recurse_package(pkg, names...) end recurse_package(pkg::String) = identify_package(pkg) -recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache()) +recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg) function recurse_package(name::String, names::String...) pkg = identify_package(name) diff --git a/test/precompile.jl b/test/precompile.jl index 92ff419bf5997..3b7ee5d6b1947 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -244,7 +244,7 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_logs (:warn, "Replacing module `$Foo_module`") begin - ms = Base._require_from_serialized(cachefile, Base.TOMLCache()) + ms = Base._require_from_serialized(cachefile) @test isa(ms, Array{Any,1}) end @@ -417,7 +417,7 @@ try """) cachefile = Base.compilecache(Base.PkgId("FooBar")) - @test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), Base.TOMLCache()) + @test cachefile == Base.compilecache_path(Base.PkgId("FooBar")) @test isfile(joinpath(cachedir, "FooBar.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector @test !isdefined(Main, :FooBar)