Use historical stdlib dependency information

ext/REPLExt/completions.jl

@@ -153,7 +153,7 @@ function complete_add_dev(options, partial, i1, i2)
  end
  comps = vcat(comps, sort(complete_remote_package(partial)))
  if !isempty(partial)
- append!(comps, filter!(startswith(partial), first.(values(Types.stdlibs()))))
+ append!(comps, filter!(startswith(partial), [info.name for info in values(Types.stdlib_infos())]))
  end
  return comps, idx, !isempty(comps)
 end

src/HistoricalStdlibs.jl

@@ -1,6 +1,17 @@
 using Base: UUID
 
-const DictStdLibs = Dict{UUID,Tuple{String,Union{VersionNumber,Nothing}}}
+struct StdlibInfo
+ name::String
+ uuid::UUID
+
+ # This can be `nothing` if it's an unregistered stdlib
+ version::Union{Nothing,VersionNumber}
+
+ deps::Vector{UUID}
+ weakdeps::Vector{UUID}
+end
+
+const DictStdLibs = Dict{UUID,StdlibInfo}
 
 # Julia standard libraries with duplicate entries removed so as to store only the
 # first release in a set of releases that all contain the same set of stdlibs.
@@ -13,32 +24,4 @@ const STDLIBS_BY_VERSION = Pair{VersionNumber, DictStdLibs}[]
 # This is a list of stdlibs that must _always_ be treated as stdlibs,
 # because they cannot be resolved in the registry; they have only ever existed within
 # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
-const UNREGISTERED_STDLIBS = DictStdLibs(
- UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => ("Base64", nothing),
- UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => ("CRC32c", nothing),
- UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => ("Dates", nothing),
- UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => ("Distributed", nothing),
- UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => ("FileWatching", nothing),
- UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => ("Future", nothing),
- UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => ("InteractiveUtils", nothing),
- UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => ("LibGit2", nothing),
- UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => ("Libdl", nothing),
- UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => ("LinearAlgebra", nothing),
- UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => ("Logging", nothing),
- UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => ("Markdown", nothing),
- UUID("a63ad114-7e13-5084-954f-fe012c677804") => ("Mmap", nothing),
- UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => ("Pkg", nothing),
- UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => ("Printf", nothing),
- UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => ("Profile", nothing),
- UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => ("REPL", nothing),
- UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => ("Random", nothing),
- UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => ("Serialization", nothing),
- UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => ("SharedArrays", nothing),
- UUID("6462fe0b-24de-5631-8697-dd941f90decc") => ("Sockets", nothing),
- UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => ("SparseArrays", nothing),
- UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => ("Statistics", nothing),
- UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => ("SuiteSparse", nothing),
- UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => ("Test", nothing),
- UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => ("UUIDs", nothing),
- UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => ("Unicode", nothing),
-)
+const UNREGISTERED_STDLIBS = DictStdLibs()

src/Operations.jl

@@ -452,7 +452,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn
  end
  end
 
- names = Dict{UUID, String}(uuid => name for (uuid, (name, version)) in stdlibs())
+ names = Dict{UUID, String}(uuid => info.name for (uuid, info) in stdlib_infos())
  # recursive search for packages which are tracking a path
  developed = collect_developed(env, pkgs)
  # But we only want to use information for those packages that we don't know about
@@ -514,7 +514,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn
  # Fixed packages are not returned by resolve (they already have their version set)
  pkg.version = vers[pkg.uuid]
  else
- name = is_stdlib(uuid) ? first(stdlibs()[uuid]) : registered_name(registries, uuid)
+ name = is_stdlib(uuid) ? stdlib_infos()[uuid].name : registered_name(registries, uuid)
  push!(pkgs, PackageSpec(;name=name, uuid=uuid, version=ver))
  end
  end
@@ -576,37 +576,26 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
  uuid in keys(fixed) && continue
  all_compat_u = get_or_make!(all_compat, uuid)
  weak_compat_u = get_or_make!(weak_compat, uuid)
-
- uuid_is_stdlib = false
- stdlib_name = ""
- stdlib_version = nothing
- if haskey(stdlibs_for_julia_version, uuid)
- uuid_is_stdlib = true
- stdlib_name, stdlib_version = stdlibs_for_julia_version[uuid]
- end
+ uuid_is_stdlib = haskey(stdlibs_for_julia_version, uuid)
 
  # If we're requesting resolution of a package that is an
  # unregistered stdlib we must special-case it here. This is further
  # complicated by the fact that we can ask this question relative to
  # a Julia version.
  if (julia_version != VERSION && is_unregistered_stdlib(uuid)) || uuid_is_stdlib
- path = Types.stdlib_path(stdlibs_for_julia_version[uuid][1])
- proj_file = projectfile_path(path; strict=true)
- @assert proj_file !== nothing
- proj = read_package(proj_file)
-
- v = something(proj.version, VERSION)
+ # We use our historical stdlib versioning data to unpack the version, deps and weakdeps of this uuid
+ stdlib_info = stdlibs_for_julia_version[uuid]
+ v = something(stdlib_info.version, VERSION)
 
- # TODO look at compat section for stdlibs?
  all_compat_u_vr = get_or_make!(all_compat_u, v)
- for (_, other_uuid) in proj.deps
+ for other_uuid in stdlib_info.deps
  push!(uuids, other_uuid)
  all_compat_u_vr[other_uuid] = VersionSpec()
  end
 
- if !isempty(proj.weakdeps)
+ if !isempty(stdlib_info.weakdeps)
  weak_all_compat_u_vr = get_or_make!(weak_compat_u, v)
- for (_, other_uuid) in proj.weakdeps
+ for other_uuid in stdlib_info.weakdeps
  push!(uuids, other_uuid)
  all_compat_u_vr[other_uuid] = VersionSpec()
  push!(weak_all_compat_u_vr, other_uuid)

src/Types.jl

@@ -21,7 +21,7 @@ using SHA
 export UUID, SHA1, VersionRange, VersionSpec,
  PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep,
  PkgError, pkgerror,
- has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
+ has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, stdlib_infos, write_env, write_env_usage, parse_toml,
  project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
  registered_name,
  manifest_info,
@@ -471,35 +471,47 @@ function load_stdlib()
  push!(FORMER_STDLIBS_UUIDS, UUID(uuid))
  continue
  end
- stdlib[UUID(uuid)] = (name, version)
+ deps = UUID.(values(get(project, "deps", Dict{String,Any}())))
+ weakdeps = UUID.(values(get(project, "weakdeps", Dict{String,Any}())))
+ stdlib[UUID(uuid)] = StdlibInfo(name, Base.UUID(uuid), version, deps, weakdeps)
  end
  return stdlib
 end
 
 function stdlibs()
+ # This maintains a compatible format for `stdlibs()`, but new code should always
+ # prefer `stdlib_infos` as it is more future-proofed, by returning a structure
+ # rather than a tuple of elements.
+ return Dict(uuid => (info.name, info.version) for (uuid, info) in stdlib_infos())
+end
+function stdlib_infos()
  if !isassigned(STDLIB)
  STDLIB[] = load_stdlib()
  end
  return STDLIB[]
 end
-is_stdlib(uuid::UUID) = uuid in keys(stdlibs())
+is_stdlib(uuid::UUID) = uuid in keys(stdlib_infos())
 # Includes former stdlibs
 function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
  return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS
 end
 
 
+function historical_stdlibs_check()
+ if isempty(STDLIBS_BY_VERSION)
+ pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant. Try `using HistoricalStdlibVersions`")
+ end
+end
+
 # Find the entry in `STDLIBS_BY_VERSION`
 # that corresponds to the requested version, and use that.
 # If we can't find one, defaults to `UNREGISTERED_STDLIBS`
 function get_last_stdlibs(julia_version::VersionNumber; use_historical_for_current_version = false)
  if !use_historical_for_current_version && julia_version == VERSION
- return stdlibs()
+ return stdlib_infos()
  end
+ historical_stdlibs_check()
  last_stdlibs = UNREGISTERED_STDLIBS
- if isempty(STDLIBS_BY_VERSION)
- pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant")
- end
  for (version, stdlibs) in STDLIBS_BY_VERSION
  if VersionNumber(julia_version.major, julia_version.minor, julia_version.patch) < version
  break
@@ -511,7 +523,10 @@ end
 # If `julia_version` is set to `nothing`, that means (essentially) treat all registered
 # stdlibs as normal packages so that we get the latest versions of everything, ignoring
 # julia compat. So we set the list of stdlibs to that of only the unregistered stdlibs.
-get_last_stdlibs(::Nothing) = UNREGISTERED_STDLIBS
+function get_last_stdlibs(::Nothing)
+ historical_stdlibs_check()
+ return UNREGISTERED_STDLIBS
+end
 
 # Allow asking if something is an stdlib for a particular version of Julia
 function is_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
@@ -535,10 +550,13 @@ function stdlib_version(uuid::UUID, julia_version::Union{VersionNumber,Nothing})
  if !(uuid in keys(last_stdlibs))
  return nothing
  end
- return last_stdlibs[uuid][2]
+ return last_stdlibs[uuid].version
 end
 
-is_unregistered_stdlib(uuid::UUID) = haskey(UNREGISTERED_STDLIBS, uuid)
+function is_unregistered_stdlib(uuid::UUID)
+ historical_stdlibs_check()
+ return haskey(UNREGISTERED_STDLIBS, uuid)
+end
 
 Context!(kw_context::Vector{Pair{Symbol,Any}})::Context =
  Context!(Context(); kw_context...)
@@ -1028,13 +1046,17 @@ function stdlib_resolve!(pkgs::AbstractVector{PackageSpec})
  for pkg in pkgs
  @assert has_name(pkg) || has_uuid(pkg)
  if has_name(pkg) && !has_uuid(pkg)
- for (uuid, (name, version)) in stdlibs()
- name == pkg.name && (pkg.uuid = uuid)
+ for (uuid, info) in stdlib_infos()
+ if info.name == pkg.name
+ pkg.uuid = uuid
+ end
  end
  end
  if !has_name(pkg) && has_uuid(pkg)
- name, version = get(stdlibs(), pkg.uuid, (nothing, nothing))
- nothing !== name && (pkg.name = name)
+ info = get(stdlib_infos(), pkg.uuid, nothing)
+ if info !== nothing
+ pkg.name = info.name
+ end
  end
  end
 end

test/Project.toml

@@ -15,4 +15,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-HistoricalStdlibVersions = "1.2"
+HistoricalStdlibVersions = "2"

test/new.jl

@@ -2999,38 +2999,37 @@ end
 
 using Pkg.Types: is_stdlib
 @testset "is_stdlib() across versions" begin
- append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+ HistoricalStdlibVersions.register!()
 
  networkoptions_uuid = UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908")
  pkg_uuid = UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f")
 
- # Assume we're running on v1.6+
+ # Test NetworkOptions across multiple versions (It became an stdlib in v1.6+, and was registered)
  @test is_stdlib(networkoptions_uuid)
  @test is_stdlib(networkoptions_uuid, v"1.6")
  @test !is_stdlib(networkoptions_uuid, v"1.5")
  @test !is_stdlib(networkoptions_uuid, v"1.0.0")
  @test !is_stdlib(networkoptions_uuid, v"0.7")
  @test !is_stdlib(networkoptions_uuid, nothing)
 
- # Pkg is an unregistered stdlib
+ # Pkg is an unregistered stdlib and has always been an stdlib
  @test is_stdlib(pkg_uuid)
  @test is_stdlib(pkg_uuid, v"1.0")
  @test is_stdlib(pkg_uuid, v"1.6")
  @test is_stdlib(pkg_uuid, v"999.999.999")
  @test is_stdlib(pkg_uuid, v"0.7")
  @test is_stdlib(pkg_uuid, nothing)
 
- empty!(Pkg.Types.STDLIBS_BY_VERSION)
-
+ HistoricalStdlibVersions.unregister!()
  # Test that we can probe for stdlibs for the current version with no STDLIBS_BY_VERSION,
  # but that we throw a PkgError if we ask for a particular julia version.
  @test is_stdlib(networkoptions_uuid)
  @test_throws Pkg.Types.PkgError is_stdlib(networkoptions_uuid, v"1.6")
 end
 
-#=
+
 @testset "Pkg.add() with julia_version" begin
- append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+ HistoricalStdlibVersions.register!()
 
  # A package with artifacts that went from normal package -> stdlib
  gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d"
@@ -3131,9 +3130,8 @@ end
  @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies))
  end
 
- empty!(Pkg.Types.STDLIBS_BY_VERSION)
+ HistoricalStdlibVersions.unregister!()
 end
-=#
 
 
 @testset "Issue #2931" begin

test/resolve.jl

@@ -683,9 +683,8 @@ end
  @test_throws ResolverError resolve_tst(deps_data, reqs_data)
 end
 
-#=
 @testset "Resolving for another version of Julia" begin
- append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+ HistoricalStdlibVersions.register!()
  temp_pkg_dir() do dir
  function find_by_name(versions, name)
  idx = findfirst(p -> p.name == name, versions)
@@ -728,15 +727,14 @@ end
  @test mpfr !== nothing
  @test mpfr.version.major == 4 && mpfr.version.minor == 0
  end
- empty!(Pkg.Types.STDLIBS_BY_VERSION)
+ HistoricalStdlibVersions.unregister!()
 end
-=#
 
 @testset "Stdlib resolve smoketest" begin
  # All stdlibs should be installable and resolvable
  temp_pkg_dir() do dir
  Pkg.activate(temp=true)
- Pkg.add(map(first, values(Pkg.Types.load_stdlib()))) # add all stdlibs
+ Pkg.add(map(x -> x.name, values(Pkg.Types.load_stdlib()))) # add all stdlibs
  iob = IOBuffer()
  Pkg.resolve(io = iob)
  str = String(take!(iob))

test/runtests.jl

@@ -20,6 +20,7 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__))
 end
 
 ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0
+ENV["HISTORICAL_STDLIB_VERSIONS_AUTO_REGISTER"]="false"
 
 logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing)
 ### Send all Pkg output to a file called Pkg.log