Allow failures of indirect dependencies during parallel precompillation

src/API.jl

@@ -894,6 +894,14 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, kwargs...
     Operations.build(ctx, pkgs, verbose)
 end
 
+function _is_stale(paths::Vector{String}, sourcepath::String, toml_c::Base.TOMLCache)
+    for path_to_try in paths
+        staledeps = Base.stale_cachefile(sourcepath, path_to_try, toml_c)
+        staledeps === true ? continue : return false
+    end
+    return true
+end
+
 _do_auto_precompile() = parse(Int, get(ENV, "JULIA_PKG_PRECOMPILE_AUTO", "0")) == 1
 
 precompile() = precompile(Context())
@@ -903,68 +911,79 @@ function precompile(ctx::Context)
     num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(Sys.CPU_THREADS + 1)))
     parallel_limiter = Base.Semaphore(num_tasks)
 
+    direct_deps = [
+        Base.PkgId(uuid, name) 
+        for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name))
+    ]
+
     man = Pkg.Types.read_manifest(ctx.env.manifest_file)
-    pkgids = [Base.PkgId(first(dep), last(dep).name) for dep in man if !Pkg.Operations.is_stdlib(first(dep))]
-    pkg_dep_uuid_lists = [collect(values(last(dep).deps)) for dep in man if !Pkg.Operations.is_stdlib(first(dep))]
-    filter!.(!is_stdlib, pkg_dep_uuid_lists)
+    deps_pair_or_nothing = Iterators.map(man) do dep
+        pkg = Base.PkgId(first(dep), last(dep).name)
+        Base.in_sysimage(pkg) && return nothing
+        deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps]
+        return pkg => filter!(!Base.in_sysimage, deps)
+    end
+    depsmap = Dict(Iterators.filter(!isnothing, deps_pair_or_nothing)) #flat map of each dep and its deps
 
     if ctx.env.pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", ctx.env.pkg.name * ".jl") )
-        push!(pkgids, Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name))
-        push!(pkg_dep_uuid_lists, collect(keys(ctx.env.project.deps)))
+        depsmap[Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name)] = [
+            Base.PkgId(last(x), first(x)) 
+            for x in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(last(x), first(x)))
+        ]
     end    
 
-    was_processed = Dict{Base.UUID,Base.Event}()
-    was_recompiled = Dict{Base.UUID,Bool}()
-    for pkgid in pkgids
-        was_processed[pkgid.uuid] = Base.Event()
-        was_recompiled[pkgid.uuid] = false
-    end
-
-    function is_stale(paths, sourcepath)
-        for path_to_try in paths::Vector{String}
-            staledeps = Base.stale_cachefile(sourcepath, path_to_try, Base.TOMLCache())
-            staledeps === true && continue
-            return false
-        end
-        return true
+    was_processed = Dict{Base.PkgId,Base.Event}()
+    was_recompiled = Dict{Base.PkgId,Bool}()
+    for pkgid in keys(depsmap)
+        was_processed[pkgid] = Base.Event()
+        was_recompiled[pkgid] = false
     end
 
     errored = false
-    @sync for (i, pkg) in pairs(pkgids)
+    toml_c = Base.TOMLCache()
+    @sync for (pkg, deps) in depsmap
         paths = Base.find_all_in_cache_path(pkg)
-        sourcepath = Base.locate_package(pkg)
+        sourcepath = Base.locate_package(pkg, toml_c)
         sourcepath === nothing && continue
         # Heuristic for when precompilation is disabled
-        occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) && continue
+        if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String))
+            notify(was_processed[pkg])
+            continue
+        end
 
         @async begin
-            for dep_uuid in pkg_dep_uuid_lists[i] # wait for deps to finish
-                wait(was_processed[dep_uuid])
+            for dep in deps # wait for deps to finish
+                wait(was_processed[dep])
             end
 
             # skip stale checking and force compilation if any dep was recompiled in this session
-            any_dep_recompiled = any(map(dep_uuid->was_recompiled[dep_uuid], pkg_dep_uuid_lists[i]))
-            if !errored && (any_dep_recompiled || is_stale(paths, sourcepath))
+            any_dep_recompiled = any(map(dep->was_recompiled[dep], deps))
+            if !errored && (any_dep_recompiled || _is_stale(paths, sourcepath, toml_c))
                 Base.acquire(parallel_limiter)
                 if errored # catch things queued before error occurred
-                    notify(was_processed[pkg.uuid])
+                    notify(was_processed[pkg])
                     Base.release(parallel_limiter)
                     return
                 end
+                is_direct_dep =  pkg in direct_deps
                 try
-                    Base.compilecache(pkg, sourcepath)
-                    was_recompiled[pkg.uuid] = true
+                    was_recompiled[pkg] = true
+                    Base.compilecache(pkg, sourcepath, is_direct_dep) # don't print errors from indirect deps
                 catch err
-                    errored = true
-                    throw(err)
+                    if is_direct_dep # only throw errors for direct dependencies (in Project)
+                        errored = true
+                        throw(err) 
+                    else
+                        @warn "Precompilation failed for indirect dependency $(pkg)"
+                    end
                 finally
-                    notify(was_processed[pkg.uuid])
+                    notify(was_processed[pkg])
                     Base.release(parallel_limiter)
                 end
             else
-                notify(was_processed[pkg.uuid])
+                notify(was_processed[pkg])
             end
-        end  
+        end
     end
     nothing
 end

src/Pkg.jl

@@ -126,7 +126,10 @@ const add = API.add
 """
     Pkg.precompile()
 
-Precompile all the dependencies of the project.
+Precompile all the dependencies of the project in parallel.
+!!! note
+    Errors will only throw when precompiling the top-level dependencies, given that 
+    not all manifest dependencies may be loaded by the top-level dependencies on the given system.
 
 !!! compat "Julia 1.3"
     This function requires at least Julia 1.3. On earlier versions