From 310a989ff4d16696b935a65aafa1d64dc38933a3 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 23 Dec 2024 21:47:48 +0000 Subject: [PATCH 1/3] precompilepkgs: respect loaded dependencies when precompiling for load --- base/loading.jl | 4 ++-- base/precompilation.jl | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8ed43e4539c20..024c4ceb356fd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1828,7 +1828,7 @@ function compilecache_path(pkg::PkgId; path = nothing isnothing(sourcepath) && error("Cannot locate source for $(repr("text/plain", pkg))") for path_to_try in cachepaths - staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags) + staledeps = stale_cachefile(sourcepath, path_to_try; ignore_loaded, requested_flags=flags) if staledeps === true continue end @@ -2649,7 +2649,7 @@ function __require_prelocked(pkg::PkgId, env) parallel_precompile_attempted = true unlock(require_lock) try - Precompilation.precompilepkgs([pkg.name]; _from_loading=true) + Precompilation.precompilepkgs([pkg.name]; _from_loading=true, ignore_loaded=false) finally lock(require_lock) end diff --git a/base/precompilation.jl b/base/precompilation.jl index c493690bab64e..160a6cb56336d 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -418,11 +418,12 @@ function precompilepkgs(pkgs::Vector{String}=String[]; io::IO=stderr, # asking for timing disables fancy mode, as timing is shown in non-fancy mode fancyprint::Bool = can_fancyprint(io) && !timing, - manifest::Bool=false,) + manifest::Bool=false, + ignore_loaded::Bool=true) # monomorphize this to avoid latency problems _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, configs isa Vector{Config} ? configs : [configs], - IOContext{IO}(io), fancyprint, manifest) + IOContext{IO}(io), fancyprint, manifest, ignore_loaded) end function _precompilepkgs(pkgs::Vector{String}, @@ -434,7 +435,8 @@ function _precompilepkgs(pkgs::Vector{String}, configs::Vector{Config}, io::IOContext{IO}, fancyprint::Bool, - manifest::Bool) + manifest::Bool, + ignore_loaded::Bool) requested_pkgs = copy(pkgs) # for understanding user intent time_start = time_ns() @@ -918,7 +920,7 @@ function _precompilepkgs(pkgs::Vector{String}, wait(was_processed[(dep,config)]) end circular = pkg in circular_deps - is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepath_cache, cachepaths, sourcepath, flags=cacheflags) + is_stale = !Base.isprecompiled(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags=cacheflags) if !circular && is_stale Base.acquire(parallel_limiter) is_project_dep = pkg in project_deps @@ -944,10 +946,10 @@ function _precompilepkgs(pkgs::Vector{String}, try # allows processes to wait if another process is precompiling a given package to # a functionally identical package cache (except for preferences, which may differ) - t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor, parallel_limiter) do + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor, parallel_limiter, ignore_loaded) do Base.with_logger(Base.NullLogger()) do - # The false here means we ignore loaded modules, so precompile for a fresh session - keep_loaded_modules = false + # whether to respect already loaded dependency versions + keep_loaded_modules = !ignore_loaded # for extensions, any extension in our direct dependencies is one we have a right to load # for packages, we may load any extension (all possible triggers are accounted for above) loadable_exts = haskey(ext_to_parent, pkg) ? filter((dep)->haskey(ext_to_parent, dep), direct_deps[pkg]) : nothing @@ -1130,7 +1132,7 @@ function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) end # Can be merged with `maybe_cachefile_lock` in loading? -function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor, parallel_limiter::Base.Semaphore) +function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor, parallel_limiter::Base.Semaphore, ignore_loaded::Bool) if !(isdefined(Base, :mkpidlock_hook) && isdefined(Base, :trymkpidlock_hook) && Base.isdefined(Base, :parse_pidfile_hook)) return f() end @@ -1158,7 +1160,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo # wait until the lock is available @invokelatest Base.mkpidlock_hook(() -> begin # double-check in case the other process crashed or the lock expired - if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed + if Base.isprecompiled(pkg; ignore_loaded, flags=cacheflags) # don't use caches for this as the env state will have changed return nothing # returning nothing indicates a process waited for another else delete!(pkgspidlocked, pkg_config) From a7ca209b31cb6eb12663532d0816ef1111607f28 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 27 Dec 2024 19:31:12 -0500 Subject: [PATCH 2/3] explain that further precompilation may be hit due to loaded deps --- base/precompilation.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 160a6cb56336d..88d3ee6caee9a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1039,9 +1039,11 @@ function _precompilepkgs(pkgs::Vector{String}, plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded == 1 ? "dependency" : "dependencies" plural2 = n_loaded == 1 ? "a different version is" : "different versions are" plural3 = n_loaded == 1 ? "" : "s" + plural4 = n_loaded == 1 ? "this package" : "these packages" print(iostr, "\n ", color_string(string(n_loaded), Base.warn_color()), - " $(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" + " $(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3). \ + Otherwise, loading dependents of $(plural4) may trigger re-precompilation to work with the unexpected version$(plural3)." ) end if !isempty(precomperr_deps) From 9a104349f7f21dc727dc902877b207e75670a72d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 29 Dec 2024 22:00:41 -0500 Subject: [PATCH 3/3] improve wording --- base/precompilation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 88d3ee6caee9a..c7dcd0af64caa 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1043,7 +1043,7 @@ function _precompilepkgs(pkgs::Vector{String}, print(iostr, "\n ", color_string(string(n_loaded), Base.warn_color()), " $(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3). \ - Otherwise, loading dependents of $(plural4) may trigger re-precompilation to work with the unexpected version$(plural3)." + Otherwise, loading dependents of $(plural4) may trigger further precompilation to work with the unexpected version$(plural3)." ) end if !isempty(precomperr_deps)