diff --git a/ext/REPLExt/completions.jl b/ext/REPLExt/completions.jl index dcdeae0cd5..aa573c91ab 100644 --- a/ext/REPLExt/completions.jl +++ b/ext/REPLExt/completions.jl @@ -153,7 +153,7 @@ function complete_add_dev(options, partial, i1, i2) end comps = vcat(comps, sort(complete_remote_package(partial))) if !isempty(partial) - append!(comps, filter!(startswith(partial), first.(values(Types.stdlibs())))) + append!(comps, filter!(startswith(partial), [info.name for info in values(Types.stdlib_infos())])) end return comps, idx, !isempty(comps) end diff --git a/src/HistoricalStdlibs.jl b/src/HistoricalStdlibs.jl index 1f50f17f95..d5b4ad5049 100644 --- a/src/HistoricalStdlibs.jl +++ b/src/HistoricalStdlibs.jl @@ -1,6 +1,17 @@ using Base: UUID -const DictStdLibs = Dict{UUID,Tuple{String,Union{VersionNumber,Nothing}}} +struct StdlibInfo + name::String + uuid::UUID + + # This can be `nothing` if it's an unregistered stdlib + version::Union{Nothing,VersionNumber} + + deps::Vector{UUID} + weakdeps::Vector{UUID} +end + +const DictStdLibs = Dict{UUID,StdlibInfo} # Julia standard libraries with duplicate entries removed so as to store only the # first release in a set of releases that all contain the same set of stdlibs. @@ -13,32 +24,4 @@ const STDLIBS_BY_VERSION = Pair{VersionNumber, DictStdLibs}[] # This is a list of stdlibs that must _always_ be treated as stdlibs, # because they cannot be resolved in the registry; they have only ever existed within # the Julia stdlib source tree, and because of that, trying to resolve them will fail. -const UNREGISTERED_STDLIBS = DictStdLibs( - UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => ("Base64", nothing), - UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => ("CRC32c", nothing), - UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => ("Dates", nothing), - UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => ("Distributed", nothing), - UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => ("FileWatching", nothing), - UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => ("Future", nothing), - UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => ("InteractiveUtils", nothing), - UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => ("LibGit2", nothing), - UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => ("Libdl", nothing), - UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => ("LinearAlgebra", nothing), - UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => ("Logging", nothing), - UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => ("Markdown", nothing), - UUID("a63ad114-7e13-5084-954f-fe012c677804") => ("Mmap", nothing), - UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => ("Pkg", nothing), - UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => ("Printf", nothing), - UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => ("Profile", nothing), - UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => ("REPL", nothing), - UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => ("Random", nothing), - UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => ("Serialization", nothing), - UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => ("SharedArrays", nothing), - UUID("6462fe0b-24de-5631-8697-dd941f90decc") => ("Sockets", nothing), - UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => ("SparseArrays", nothing), - UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => ("Statistics", nothing), - UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => ("SuiteSparse", nothing), - UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => ("Test", nothing), - UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => ("UUIDs", nothing), - UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => ("Unicode", nothing), -) +const UNREGISTERED_STDLIBS = DictStdLibs() diff --git a/src/Operations.jl b/src/Operations.jl index a8ca93df2c..595db16044 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -452,7 +452,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn end end - names = Dict{UUID, String}(uuid => name for (uuid, (name, version)) in stdlibs()) + names = Dict{UUID, String}(uuid => info.name for (uuid, info) in stdlib_infos()) # recursive search for packages which are tracking a path developed = collect_developed(env, pkgs) # But we only want to use information for those packages that we don't know about @@ -514,7 +514,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn # Fixed packages are not returned by resolve (they already have their version set) pkg.version = vers[pkg.uuid] else - name = is_stdlib(uuid) ? first(stdlibs()[uuid]) : registered_name(registries, uuid) + name = is_stdlib(uuid) ? stdlib_infos()[uuid].name : registered_name(registries, uuid) push!(pkgs, PackageSpec(;name=name, uuid=uuid, version=ver)) end end @@ -576,37 +576,26 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} uuid in keys(fixed) && continue all_compat_u = get_or_make!(all_compat, uuid) weak_compat_u = get_or_make!(weak_compat, uuid) - - uuid_is_stdlib = false - stdlib_name = "" - stdlib_version = nothing - if haskey(stdlibs_for_julia_version, uuid) - uuid_is_stdlib = true - stdlib_name, stdlib_version = stdlibs_for_julia_version[uuid] - end + uuid_is_stdlib = haskey(stdlibs_for_julia_version, uuid) # If we're requesting resolution of a package that is an # unregistered stdlib we must special-case it here. This is further # complicated by the fact that we can ask this question relative to # a Julia version. if (julia_version != VERSION && is_unregistered_stdlib(uuid)) || uuid_is_stdlib - path = Types.stdlib_path(stdlibs_for_julia_version[uuid][1]) - proj_file = projectfile_path(path; strict=true) - @assert proj_file !== nothing - proj = read_package(proj_file) - - v = something(proj.version, VERSION) + # We use our historical stdlib versioning data to unpack the version, deps and weakdeps of this uuid + stdlib_info = stdlibs_for_julia_version[uuid] + v = something(stdlib_info.version, VERSION) - # TODO look at compat section for stdlibs? all_compat_u_vr = get_or_make!(all_compat_u, v) - for (_, other_uuid) in proj.deps + for other_uuid in stdlib_info.deps push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() end - if !isempty(proj.weakdeps) + if !isempty(stdlib_info.weakdeps) weak_all_compat_u_vr = get_or_make!(weak_compat_u, v) - for (_, other_uuid) in proj.weakdeps + for other_uuid in stdlib_info.weakdeps push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() push!(weak_all_compat_u_vr, other_uuid) diff --git a/src/Types.jl b/src/Types.jl index 24d60cf7ed..4959160d76 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -21,7 +21,7 @@ using SHA export UUID, SHA1, VersionRange, VersionSpec, PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep, PkgError, pkgerror, - has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml, + has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, stdlib_infos, write_env, write_env_usage, parse_toml, project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved, registered_name, manifest_info, @@ -471,35 +471,47 @@ function load_stdlib() push!(FORMER_STDLIBS_UUIDS, UUID(uuid)) continue end - stdlib[UUID(uuid)] = (name, version) + deps = UUID.(values(get(project, "deps", Dict{String,Any}()))) + weakdeps = UUID.(values(get(project, "weakdeps", Dict{String,Any}()))) + stdlib[UUID(uuid)] = StdlibInfo(name, Base.UUID(uuid), version, deps, weakdeps) end return stdlib end function stdlibs() + # This maintains a compatible format for `stdlibs()`, but new code should always + # prefer `stdlib_infos` as it is more future-proofed, by returning a structure + # rather than a tuple of elements. + return Dict(uuid => (info.name, info.version) for (uuid, info) in stdlib_infos()) +end +function stdlib_infos() if !isassigned(STDLIB) STDLIB[] = load_stdlib() end return STDLIB[] end -is_stdlib(uuid::UUID) = uuid in keys(stdlibs()) +is_stdlib(uuid::UUID) = uuid in keys(stdlib_infos()) # Includes former stdlibs function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing}) return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS end +function historical_stdlibs_check() + if isempty(STDLIBS_BY_VERSION) + pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant. Try `using HistoricalStdlibVersions`") + end +end + # Find the entry in `STDLIBS_BY_VERSION` # that corresponds to the requested version, and use that. # If we can't find one, defaults to `UNREGISTERED_STDLIBS` function get_last_stdlibs(julia_version::VersionNumber; use_historical_for_current_version = false) if !use_historical_for_current_version && julia_version == VERSION - return stdlibs() + return stdlib_infos() end + historical_stdlibs_check() last_stdlibs = UNREGISTERED_STDLIBS - if isempty(STDLIBS_BY_VERSION) - pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant") - end for (version, stdlibs) in STDLIBS_BY_VERSION if VersionNumber(julia_version.major, julia_version.minor, julia_version.patch) < version break @@ -511,7 +523,10 @@ end # If `julia_version` is set to `nothing`, that means (essentially) treat all registered # stdlibs as normal packages so that we get the latest versions of everything, ignoring # julia compat. So we set the list of stdlibs to that of only the unregistered stdlibs. -get_last_stdlibs(::Nothing) = UNREGISTERED_STDLIBS +function get_last_stdlibs(::Nothing) + historical_stdlibs_check() + return UNREGISTERED_STDLIBS +end # Allow asking if something is an stdlib for a particular version of Julia function is_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing}) @@ -535,10 +550,13 @@ function stdlib_version(uuid::UUID, julia_version::Union{VersionNumber,Nothing}) if !(uuid in keys(last_stdlibs)) return nothing end - return last_stdlibs[uuid][2] + return last_stdlibs[uuid].version end -is_unregistered_stdlib(uuid::UUID) = haskey(UNREGISTERED_STDLIBS, uuid) +function is_unregistered_stdlib(uuid::UUID) + historical_stdlibs_check() + return haskey(UNREGISTERED_STDLIBS, uuid) +end Context!(kw_context::Vector{Pair{Symbol,Any}})::Context = Context!(Context(); kw_context...) @@ -1028,13 +1046,17 @@ function stdlib_resolve!(pkgs::AbstractVector{PackageSpec}) for pkg in pkgs @assert has_name(pkg) || has_uuid(pkg) if has_name(pkg) && !has_uuid(pkg) - for (uuid, (name, version)) in stdlibs() - name == pkg.name && (pkg.uuid = uuid) + for (uuid, info) in stdlib_infos() + if info.name == pkg.name + pkg.uuid = uuid + end end end if !has_name(pkg) && has_uuid(pkg) - name, version = get(stdlibs(), pkg.uuid, (nothing, nothing)) - nothing !== name && (pkg.name = name) + info = get(stdlib_infos(), pkg.uuid, nothing) + if info !== nothing + pkg.name = info.name + end end end end diff --git a/test/Project.toml b/test/Project.toml index 0f0fd0e410..0922624374 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -15,4 +15,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -HistoricalStdlibVersions = "1.2" +HistoricalStdlibVersions = "2" diff --git a/test/new.jl b/test/new.jl index 6318beba49..50657b5df8 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2999,12 +2999,12 @@ end using Pkg.Types: is_stdlib @testset "is_stdlib() across versions" begin - append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION) + HistoricalStdlibVersions.register!() networkoptions_uuid = UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") pkg_uuid = UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") - # Assume we're running on v1.6+ + # Test NetworkOptions across multiple versions (It became an stdlib in v1.6+, and was registered) @test is_stdlib(networkoptions_uuid) @test is_stdlib(networkoptions_uuid, v"1.6") @test !is_stdlib(networkoptions_uuid, v"1.5") @@ -3012,7 +3012,7 @@ using Pkg.Types: is_stdlib @test !is_stdlib(networkoptions_uuid, v"0.7") @test !is_stdlib(networkoptions_uuid, nothing) - # Pkg is an unregistered stdlib + # Pkg is an unregistered stdlib and has always been an stdlib @test is_stdlib(pkg_uuid) @test is_stdlib(pkg_uuid, v"1.0") @test is_stdlib(pkg_uuid, v"1.6") @@ -3020,17 +3020,16 @@ using Pkg.Types: is_stdlib @test is_stdlib(pkg_uuid, v"0.7") @test is_stdlib(pkg_uuid, nothing) - empty!(Pkg.Types.STDLIBS_BY_VERSION) - + HistoricalStdlibVersions.unregister!() # Test that we can probe for stdlibs for the current version with no STDLIBS_BY_VERSION, # but that we throw a PkgError if we ask for a particular julia version. @test is_stdlib(networkoptions_uuid) @test_throws Pkg.Types.PkgError is_stdlib(networkoptions_uuid, v"1.6") end -#= + @testset "Pkg.add() with julia_version" begin - append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION) + HistoricalStdlibVersions.register!() # A package with artifacts that went from normal package -> stdlib gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" @@ -3131,9 +3130,8 @@ end @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies)) end - empty!(Pkg.Types.STDLIBS_BY_VERSION) + HistoricalStdlibVersions.unregister!() end -=# @testset "Issue #2931" begin diff --git a/test/resolve.jl b/test/resolve.jl index f45e12b20f..a49e9a9139 100644 --- a/test/resolve.jl +++ b/test/resolve.jl @@ -683,9 +683,8 @@ end @test_throws ResolverError resolve_tst(deps_data, reqs_data) end -#= @testset "Resolving for another version of Julia" begin - append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION) + HistoricalStdlibVersions.register!() temp_pkg_dir() do dir function find_by_name(versions, name) idx = findfirst(p -> p.name == name, versions) @@ -728,15 +727,14 @@ end @test mpfr !== nothing @test mpfr.version.major == 4 && mpfr.version.minor == 0 end - empty!(Pkg.Types.STDLIBS_BY_VERSION) + HistoricalStdlibVersions.unregister!() end -=# @testset "Stdlib resolve smoketest" begin # All stdlibs should be installable and resolvable temp_pkg_dir() do dir Pkg.activate(temp=true) - Pkg.add(map(first, values(Pkg.Types.load_stdlib()))) # add all stdlibs + Pkg.add(map(x -> x.name, values(Pkg.Types.load_stdlib()))) # add all stdlibs iob = IOBuffer() Pkg.resolve(io = iob) str = String(take!(iob)) diff --git a/test/runtests.jl b/test/runtests.jl index 61300f9656..6e418a74ae 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,7 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__)) end ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0 +ENV["HISTORICAL_STDLIB_VERSIONS_AUTO_REGISTER"]="false" logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing) ### Send all Pkg output to a file called Pkg.log