From ee2d510549be8eba87b0ea5d67b8a751390a7751 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 22 Mar 2024 20:14:14 +0100 Subject: [PATCH 1/6] collect e.g. weak deps from project even if it is not a package (#3852) * collect e.g. weak deps from project even if it is not a package (cherry picked from commit 9c6356fa9d838388cc308cfedaec6cd38fdf00ba) --- src/Operations.jl | 38 +++++++++---------- src/Types.jl | 2 +- test/extensions.jl | 12 ++++++ .../TestWeakDepProject/Project.toml | 9 +++++ 4 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 test/test_packages/TestWeakDepProject/Project.toml diff --git a/src/Operations.jl b/src/Operations.jl index a0da2c5050..db729e8454 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -246,17 +246,14 @@ function reset_all_compat!(proj::Project) return nothing end -function collect_project(pkg::PackageSpec, path::String) +function collect_project(pkg::Union{PackageSpec, Nothing}, path::String) deps = PackageSpec[] weakdeps = Set{UUID}() project_file = projectfile_path(path; strict=true) - if project_file === nothing - pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`") - end - project = read_package(project_file) + project = project_file === nothing ? Project() : read_project(project_file) julia_compat = get_compat(project, "julia") if !isnothing(julia_compat) && !(VERSION in julia_compat) - pkgerror("julia version requirement from Project.toml's compat section not satisfied for package $(err_rep(pkg)) at `$path`") + pkgerror("julia version requirement from Project.toml's compat section not satisfied for package at `$path`") end for (name, uuid) in project.deps path, repo = get_path_repo(project, name) @@ -268,11 +265,13 @@ function collect_project(pkg::PackageSpec, path::String) push!(deps, PackageSpec(name, uuid, vspec)) push!(weakdeps, uuid) end - if project.version !== nothing - pkg.version = project.version - else - # @warn("project file for $(pkg.name) is missing a `version` entry") - pkg.version = VersionNumber(0) + if pkg !== nothing + if project.version !== nothing + pkg.version = project.version + else + # @warn("project file for $(pkg.name) is missing a `version` entry") + pkg.version = VersionNumber(0) + end end return deps, weakdeps end @@ -310,13 +309,13 @@ end function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String}) deps_map = Dict{UUID,Vector{PackageSpec}}() weak_map = Dict{UUID,Set{UUID}}() - if env.pkg !== nothing - pkg = env.pkg - deps, weakdeps = collect_project(pkg, dirname(env.project_file)) - deps_map[pkg.uuid] = deps - weak_map[pkg.uuid] = weakdeps - names[pkg.uuid] = pkg.name - end + + uuid = Types.project_uuid(env) + deps, weakdeps = collect_project(env.pkg, dirname(env.project_file)) + deps_map[uuid] = deps + weak_map[uuid] = weakdeps + names[uuid] = env.pkg === nothing ? "project" : env.pkg.name + for pkg in pkgs # add repo package if necessary if (pkg.repo.rev !== nothing || pkg.repo.source !== nothing) && pkg.tree_hash === nothing @@ -346,7 +345,8 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU idx = findfirst(pkg -> pkg.uuid == uuid, pkgs) fix_pkg = pkgs[idx] end - fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, weak_map[uuid]) + fixpkgversion = fix_pkg === nothing ? v"0.0.0" : fix_pkg.version + fixed[uuid] = Resolve.Fixed(fixpkgversion, q, weak_map[uuid]) end return fixed end diff --git a/src/Types.jl b/src/Types.jl index 6bbdca5ad5..4c44658357 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -412,7 +412,7 @@ Base.@kwdef mutable struct Context julia_version::Union{VersionNumber,Nothing} = VERSION end -project_uuid(env::EnvCache) = env.pkg === nothing ? nothing : env.pkg.uuid +project_uuid(env::EnvCache) = env.pkg === nothing ? Base.dummy_uuid(env.project_file) : env.pkg.uuid collides_with_project(env::EnvCache, pkg::PackageSpec) = is_project_name(env, pkg.name) || is_project_uuid(env, pkg.uuid) is_project(env::EnvCache, pkg::PackageSpec) = is_project_uuid(env, pkg.uuid) diff --git a/test/extensions.jl b/test/extensions.jl index a53ae2131d..2d0b4f7781 100644 --- a/test/extensions.jl +++ b/test/extensions.jl @@ -1,5 +1,6 @@ using .Utils using Test +using UUIDs @testset "weak deps" begin he_root = joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasExtensions.jl") @@ -93,4 +94,15 @@ using Test @test proj.extras == Dict{String, Base.UUID}("Example" => Base.UUID("7876af07-990d-54b4-ab0e-23690620f79a")) end end + + isolate(loaded_depot=false) do + mktempdir() do dir + Pkg.Registry.add("General") + path = joinpath(@__DIR__, "test_packages", "TestWeakDepProject") + cp(path, joinpath(dir, "TestWeakDepProject")) + Pkg.activate(joinpath(dir, "TestWeakDepProject")) + Pkg.resolve() + @test Pkg.dependencies()[UUID("2ab3a3ac-af41-5b50-aa03-7779005ae688")].version == v"0.3.26" + end + end end diff --git a/test/test_packages/TestWeakDepProject/Project.toml b/test/test_packages/TestWeakDepProject/Project.toml new file mode 100644 index 0000000000..c856f60f08 --- /dev/null +++ b/test/test_packages/TestWeakDepProject/Project.toml @@ -0,0 +1,9 @@ +[deps] +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + +[weakdeps] +LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688" + +[compat] +ForwardDiff = "=0.10.36" +LogExpFunctions = "=0.3.26" From 0b2397089d42dd84dd972c0b471f27703c6a85c4 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 10 Apr 2024 11:17:16 +0200 Subject: [PATCH 2/6] make `add` and `dev` on a package remove it from the set of weak dependencies (#3865) * make `add` and `dev` on a package remove it from the weakdeps section (cherry picked from commit 88c38b2cd00e93e77c78ab952ffb6a06e6e1828b) --- src/Operations.jl | 15 +++++++++++---- test/extensions.jl | 9 +++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index db729e8454..f53eec7f0b 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1403,6 +1403,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); assert_can_add(ctx, pkgs) # load manifest data for (i, pkg) in pairs(pkgs) + delete!(ctx.env.project.weakdeps, pkg.name) entry = manifest_info(ctx.env.manifest, pkg.uuid) is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps]) source_path, source_repo = get_path_repo(ctx.env.project, pkg.name) @@ -1467,6 +1468,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; assert_can_add(ctx, pkgs) # no need to look at manifest.. dev will just nuke whatever is there before for pkg in pkgs + delete!(ctx.env.project.weakdeps, pkg.name) ctx.env.project.deps[pkg.name] = pkg.uuid end # resolve & apply package versions @@ -2288,6 +2290,7 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache) manifest = env.manifest manifest_info = get(manifest, pkg.uuid, nothing) manifest_info === nothing && return nothing + depses = manifest_info.deps weakdepses = manifest_info.weakdeps exts = manifest_info.exts if !isempty(weakdepses) && !isempty(exts) @@ -2299,10 +2302,14 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache) # Check if deps are loaded extdeps_info= Tuple{String, Bool}[] for extdep in extdeps - haskey(weakdepses, extdep) || - pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a m", - "alformed Project.toml, the extension package $extdep is not listed in [weakdeps]") - uuid = weakdepses[extdep] + if !(haskey(weakdepses, extdep) || haskey(depses, extdep)) + pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a malformed Project.toml, ", + "the extension package $extdep is not listed in [weakdeps] or [deps]") + end + uuid = get(weakdepses, extdep, nothing) + if uuid === nothing + uuid = depses[extdep] + end loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, extdep)) push!(extdeps_info, (extdep, loaded)) end diff --git a/test/extensions.jl b/test/extensions.jl index 2d0b4f7781..f7d7ab26b9 100644 --- a/test/extensions.jl +++ b/test/extensions.jl @@ -103,6 +103,15 @@ using UUIDs Pkg.activate(joinpath(dir, "TestWeakDepProject")) Pkg.resolve() @test Pkg.dependencies()[UUID("2ab3a3ac-af41-5b50-aa03-7779005ae688")].version == v"0.3.26" + + # Check that explicitly adding a package that is a weak dep removes it from the set of weak deps + ctx = Pkg.Types.Context() + @test "LogExpFunctions" in keys(ctx.env.project.weakdeps) + @test !("LogExpFunctions" in keys(ctx.env.project.deps)) + Pkg.add("LogExpFunctions") + ctx = Pkg.Types.Context() + @test "LogExpFunctions" in keys(ctx.env.project.deps) + @test !("LogExpFunctions" in keys(ctx.env.project.weakdeps)) end end end From 72dc85e80c4e1f53af60f11596d2ba5708dc72e4 Mon Sep 17 00:00:00 2001 From: Peter Simon Date: Sat, 14 Sep 2024 06:17:04 -0700 Subject: [PATCH 3/6] Tweak sentence syntax in getting-started.md (#4020) (cherry picked from commit e5f400b2490d5c29c43754dbeb404783050cf451) --- docs/src/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 1539cda6cd..fe1f652dd3 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -154,7 +154,7 @@ We can see that the `tutorial` environment now contains `Example` and `JSON`. If you have the same package (at the same version) installed in multiple environments, the package will only be downloaded and stored on the hard drive once. This makes environments - very lightweight and effectively free to create. Only using the default + very lightweight and effectively free to create. Using only the default environment with a huge number of packages in it is a common beginners mistake in Julia. Learning how to use environments effectively will improve your experience with Julia packages. From 1475b628ae45b86cdce46a7105dbee5b867dd8e9 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Thu, 19 Sep 2024 07:37:17 -0400 Subject: [PATCH 4/6] update package extension naming docs (#4000) (cherry picked from commit 5fbfa125045ce3e68ce10bf9fc1727bb3232c123) --- docs/src/creating-packages.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 4959260733..eb4e66edf6 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -416,7 +416,7 @@ Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" # name of extension to the left # extension dependencies required to load the extension to the right # use a list for multiple extension dependencies -PlottingContourExt = "Contour" +ContourExt = "Contour" [compat] Contour = "0.6.2" @@ -433,9 +433,9 @@ end end # module ``` -`ext/PlottingContourExt.jl` (can also be in `ext/PlottingContourExt/PlottingContourExt.jl`): +`ext/ContourExt.jl` (can also be in `ext/ContourExt/ContourExt.jl`): ```julia -module PlottingContourExt # Should be same name as the file (just like a normal package) +module ContourExt # Should be same name as the file (just like a normal package) using Plotting, Contour @@ -446,8 +446,8 @@ end end # module ``` -Extensions can have any arbitrary name (here `PlottingContourExt`), but using something similar to the format of -this example that makes the extended functionality and dependency of the extension clear is likely a good idea. +Extensions can have arbitrary names (here `ContourExt`), following the format of this example is likely a good idea for extensions with a single dependency. +In `Pkg` output, extension names are always shown together with their parent package name. !!! compat Often you will put the extension dependencies into the `test` target so they are loaded when running e.g. `Pkg.test()`. On earlier Julia versions @@ -461,8 +461,8 @@ this example that makes the extended functionality and dependency of the extensi ### Behavior of extensions -A user that depends only on `Plotting` will not pay the cost of the "extension" inside the `PlottingContourExt` module. -It is only when the `Contour` package actually gets loaded that the `PlottingContourExt` extension is loaded too +A user that depends only on `Plotting` will not pay the cost of the "extension" inside the `ContourExt` module. +It is only when the `Contour` package actually gets loaded that the `ContourExt` extension is loaded too and provides the new functionality. In our example, the new functionality is an additional _method_, which we add to an existing _function_ from the parent package `Plotting`. @@ -474,17 +474,17 @@ function plot end ``` !!! note - If one considers `PlottingContourExt` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is - [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy) since `PlottingContourExt` _owns_ neither the function `Plotting.plot` nor the type `Contour.ContourCollection`. + If one considers `ContourExt` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is + [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy) since `ContourExt` _owns_ neither the function `Plotting.plot` nor the type `Contour.ContourCollection`. However, for extensions, it is ok to assume that the extension owns the functions in its parent package. In other situations, one may need to define new symbols in the extension (types, structs, functions, etc.) instead of reusing those from the parent package. -Such symbols are created in a separate module corresponding to the extension, namely `PlottingContourExt`, and thus not in `Plotting` itself. +Such symbols are created in a separate module corresponding to the extension, namely `ContourExt`, and thus not in `Plotting` itself. If extension symbols are needed in the parent package, one must call `Base.get_extension` to retrieve them. -Here is an example showing how a custom type defined in `PlottingContourExt` can be accessed in `Plotting`: +Here is an example showing how a custom type defined in `ContourExt` can be accessed in `Plotting`: ```julia -ext = Base.get_extension(@__MODULE__, :PlottingContourExt) +ext = Base.get_extension(@__MODULE__, :ContourExt) if !isnothing(ext) ContourPlotType = ext.ContourPlotType end @@ -512,7 +512,7 @@ This is done by making the following changes (using the example above): @static if !isdefined(Base, :get_extension) function __init__() - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/PlottingContourExt.jl") + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl") end end ``` @@ -526,11 +526,11 @@ This is done by making the following changes (using the example above): # Other init functionality here @static if !isdefined(Base, :get_extension) - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/PlottingContourExt.jl") + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl") end end ``` -- Make the following change in the conditionally-loaded code: +- Make the following change in the conditionally-loaded code in `ContourExt.jl`: ```julia isdefined(Base, :get_extension) ? (using Contour) : (using ..Contour) ``` @@ -549,7 +549,7 @@ This is done by making the following changes (using the example above): - Add the following to your main package file (typically at the bottom): ```julia if !isdefined(Base, :get_extension) - include("../ext/PlottingContourExt.jl") + include("../ext/ContourExt.jl") end ``` From df38587fb8146821c4a89e589a1416ef89ae6af9 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 22 Sep 2024 08:13:41 -0400 Subject: [PATCH 5/6] warn if General is installed via the old slow methods (#4022) * warn if General is installed via the old slow methods * add env var disable (cherry picked from commit acc54fc02d313c4917c9b3bb782c0d9e92e0555b) --- src/Registry/Registry.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 8d5137daba..d5e938baa1 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -447,6 +447,15 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true, registry_update_log[string(reg.uuid)] = now() @label done_tarball_read else + if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) + @info """ + The General registry is installed via unpacked tarball. + Consider reinstalling it via the newer faster direct from + tarball format by running: + pkg> registry rm General; registry add General + + """ maxlog=1 + end mktempdir() do tmp try download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io) @@ -465,6 +474,14 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true, end elseif isdir(joinpath(reg.path, ".git")) printpkgstyle(io, :Updating, "registry at " * regpath) + if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) + @info """ + The General registry is installed via git. Consider reinstalling it via + the newer faster direct from tarball format by running: + pkg> registry rm General; registry add General + + """ maxlog=1 + end LibGit2.with(LibGit2.GitRepo(reg.path)) do repo if LibGit2.isdirty(repo) push!(errors, (regpath, "registry dirty")) From 76eaa4caa437d1ee9e98738479980546dec518d1 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:40:34 -0300 Subject: [PATCH 6/6] Fix julia#55850 by using safe_realpath instead of abspath in projname (#4025) * Update REPLExt.jl * Add test * Update test/repl.jl Co-authored-by: Ian Butterworth * Update test/repl.jl Co-authored-by: Ian Butterworth * Fix windows tests --------- Co-authored-by: Ian Butterworth (cherry picked from commit 2ad377f1398bfa4f99a187414d1270009b05daec) --- ext/REPLExt/REPLExt.jl | 4 ++-- test/repl.jl | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ext/REPLExt/REPLExt.jl b/ext/REPLExt/REPLExt.jl index 485e9110f2..a88cf11427 100644 --- a/ext/REPLExt/REPLExt.jl +++ b/ext/REPLExt/REPLExt.jl @@ -7,7 +7,7 @@ import .REPL: LineEdit, REPLCompletions, TerminalMenus import Pkg import .Pkg: linewrap, pathrepr, compat, can_fancyprint, printpkgstyle, PKGMODE_PROJECT -using .Pkg: Types, Operations, API, Registry, Resolve, REPLMode +using .Pkg: Types, Operations, API, Registry, Resolve, REPLMode, safe_realpath using .REPLMode: Statement, CommandSpec, Command, prepare_cmd, tokenize, core_parse, SPECS, api_options, parse_option, api_options, is_opt, wrap_option @@ -47,7 +47,7 @@ function projname(project_file::String) end for depot in Base.DEPOT_PATH envdir = joinpath(depot, "environments") - if startswith(abspath(project_file), abspath(envdir)) + if startswith(safe_realpath(project_file), safe_realpath(envdir)) return "@" * name end end diff --git a/test/repl.jl b/test/repl.jl index 6321cda65f..b0d729dd92 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -734,4 +734,14 @@ end end end +@testset "JuliaLang/julia #55850" begin + tmp_55850 = mktempdir() + tmp_sym_link = joinpath(tmp_55850, "sym") + symlink(tmp_55850, tmp_sym_link; dir_target=true) + withenv("JULIA_DEPOT_PATH" => tmp_sym_link * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do + prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`) + @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> " + end +end + end # module