Allow failures of indirect dependencies during parallel precompillation

src/API.jl

@@ -894,71 +894,88 @@ end
 
 precompile() = precompile(Context())
 function precompile(ctx::Context)
+
+    function is_stale(paths, sourcepath)
+        toml_c = Base.TOMLCache()
+        for path_to_try in paths::Vector{String}
+            staledeps = Base.stale_cachefile(sourcepath, path_to_try, toml_c)
+            staledeps === true ? continue : return false
+        end
+        return true
+    end
+
     printpkgstyle(ctx, :Precompiling, "project...")
 
     num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(Sys.CPU_THREADS + 1)))
     parallel_limiter = Base.Semaphore(num_tasks)
 
+    direct_deps = [
+        Base.PkgId(uuid, name) 
+        for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name))
+    ]
+
     man = Pkg.Types.read_manifest(ctx.env.manifest_file)
-    pkgids = [Base.PkgId(first(dep), last(dep).name) for dep in man if !Pkg.Operations.is_stdlib(first(dep))]
-    pkg_dep_uuid_lists = [collect(values(last(dep).deps)) for dep in man if !Pkg.Operations.is_stdlib(first(dep))]
-    filter!.(!is_stdlib, pkg_dep_uuid_lists)
+    deps_pair_or_nothing = Iterators.map(man) do dep
+        pkg = Base.PkgId(first(dep), last(dep).name)
+        Base.in_sysimage(pkg) && return nothing
+        deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps]
+        return pkg => filter!(!Base.in_sysimage, deps)
+    end
+    depsmap = Dict(Iterators.filter(!isnothing, deps_pair_or_nothing)) #flat map of each dep and its deps
 
     if ctx.env.pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", ctx.env.pkg.name * ".jl") )
-        push!(pkgids, Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name))
-        push!(pkg_dep_uuid_lists, collect(keys(ctx.env.project.deps)))
+        depsmap[Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name)] = [
+            Base.PkgId(last(x), first(x)) 
+            for x in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(last(x), first(x)))
+        ]
     end    
 
-    was_processed = Dict{Base.UUID,Base.Event}()
-    was_recompiled = Dict{Base.UUID,Bool}()
-    for pkgid in pkgids
-        was_processed[pkgid.uuid] = Base.Event()
-        was_recompiled[pkgid.uuid] = false
-    end
-
-    function is_stale(paths, sourcepath)
-        for path_to_try in paths::Vector{String}
-            staledeps = Base.stale_cachefile(sourcepath, path_to_try, Base.TOMLCache())
-            staledeps === true && continue
-            return false
-        end
-        return true
+    was_processed = Dict{Base.PkgId,Base.Event}()
+    was_recompiled = Dict{Base.PkgId,Bool}()
+    for pkgid in keys(depsmap)
+        was_processed[pkgid] = Base.Event()
+        was_recompiled[pkgid] = false
     end
 
     errored = false
-    @sync for (i, pkg) in pairs(pkgids)
+    @sync for (pkg, deps) in depsmap
         paths = Base.find_all_in_cache_path(pkg)
         sourcepath = Base.locate_package(pkg)
         sourcepath === nothing && continue
         # Heuristic for when precompilation is disabled
         occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) && continue
 
         @async begin
-            for dep_uuid in pkg_dep_uuid_lists[i] # wait for deps to finish
-                wait(was_processed[dep_uuid])
+            for dep in deps # wait for deps to finish
+                wait(was_processed[dep])
             end
 
             # skip stale checking and force compilation if any dep was recompiled in this session
-            any_dep_recompiled = any(map(dep_uuid->was_recompiled[dep_uuid], pkg_dep_uuid_lists[i]))
+            any_dep_recompiled = any(map(dep->was_recompiled[dep], deps))
             if !errored && (any_dep_recompiled || is_stale(paths, sourcepath))
                 Base.acquire(parallel_limiter)
                 if errored # catch things queued before error occurred
-                    notify(was_processed[pkg.uuid])
+                    notify(was_processed[pkg])
                     Base.release(parallel_limiter)
                     return
                 end
+                is_direct_dep =  pkg in direct_deps
                 try
-                    Base.compilecache(pkg, sourcepath)
-                    was_recompiled[pkg.uuid] = true
+                    was_recompiled[pkg] = true
+                    Base.compilecache(pkg, sourcepath, is_direct_dep) # don't print errors from indirect deps
                 catch err
-                    errored = true
-                    throw(err)
+                    if is_direct_dep # only throw errors for direct dependencies (in Project)
+                        errored = true
+                        throw(err) 
+                    else
+                        @warn "Precompilation failed for indirect dependency $(pkg)"
+                    end
                 finally
-                    notify(was_processed[pkg.uuid])
+                    notify(was_processed[pkg])
                     Base.release(parallel_limiter)
                 end
             else
-                notify(was_processed[pkg.uuid])
+                notify(was_processed[pkg])
             end
         end  
     end

src/Pkg.jl

@@ -126,7 +126,10 @@ const add = API.add
 """
     Pkg.precompile()
 
-Precompile all the dependencies of the project.
+Precompile all the dependencies of the project in parallel.
+!!! note
+    Errors will only throw when precompiling the top-level dependencies, given that 
+    not all manifest dependencies may be loaded by the top-level dependencies on the given system.
 
 !!! compat "Julia 1.3"
     This function requires at least Julia 1.3. On earlier versions