allow extensions to trigger from packages in [deps] (#54009)

There is a use case where you have a weak dependency (for one of your
extensions) that is misbehaving and you quickly want to try debug that
issue. A workflow that feels reasonable for this could be:

```
pkg> dev WeakDependency

# do some fixes in this dependency

julia> using Package, WeakDependency
# this loads the extension of Package triggered by loading WeakDependency

# check that things work ok now
```

This doesn't work right now for two reasons:

1. Doing the `dev WeakDependency` will add the dependency to `[deps]`
but not remove it from `[weakdeps]` which means you all of a sudden are
in the scenario described in
https://pkgdocs.julialang.org/v1/creating-packages/#Transition-from-normal-dependency-to-extension
which is not what is desired.
2. The extension will not actually load because you can right now only
trigger extensions from weak deps getting loaded, not from deps getting
loaded.

Point 1. is fixed by JuliaLang/Pkg.jl#3865
Point 2. is fixed by this PR.

base/loading.jl

@@ -1388,13 +1388,12 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi
  proj_pkg = project_file_name_uuid(implicit_project_file, pkg.name)
  if pkg == proj_pkg
  d_proj = parsed_toml(implicit_project_file)
- weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
  extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
  extensions === nothing && return
- weakdeps === nothing && return
- if weakdeps isa Dict{String, Any}
-  return _insert_extension_triggers(pkg, extensions, weakdeps)
- end
+ weakdeps = get(Dict{String, Any}, d_proj, "weakdeps")::Dict{String,Any}
+ deps = get(Dict{String, Any}, d_proj, "deps")::Dict{String,Any}
+ total_deps = merge(weakdeps, deps)
+ return _insert_extension_triggers(pkg, extensions, total_deps)
  end
 
  # Now look in manifest
@@ -1409,35 +1408,43 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi
  uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
  uuid === nothing && continue
  if UUID(uuid) == pkg.uuid
- weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
  extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
  extensions === nothing && return
- weakdeps === nothing && return
- if weakdeps isa Dict{String, Any}
- return _insert_extension_triggers(pkg, extensions, weakdeps)
+ weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}}
+ deps = get(Dict{String, Any}, entry, "deps")::Union{Vector{String}, Dict{String,Any}}
+
+ function expand_deps_list(deps′::Vector{String})
+ deps′_expanded = Dict{String, Any}()
+ for (dep_name, entries) in d
+ dep_name in deps′ || continue
+ entries::Vector{Any}
+ if length(entries) != 1
+ error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))")
+ end
+ entry = first(entries)::Dict{String, Any}
+ uuid = entry["uuid"]::String
+ deps′_expanded[dep_name] = uuid
+ end
+ return deps′_expanded
  end
 
- d_weakdeps = Dict{String, Any}()
- for (dep_name, entries) in d
- dep_name in weakdeps || continue
- entries::Vector{Any}
- if length(entries) != 1
- error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))")
- end
- entry = first(entries)::Dict{String, Any}
- uuid = entry["uuid"]::String
- d_weakdeps[dep_name] = uuid
+ if weakdeps isa Vector{String}
+ weakdeps = expand_deps_list(weakdeps)
+ end
+ if deps isa Vector{String}
+ deps = expand_deps_list(deps)
  end
- @assert length(d_weakdeps) == length(weakdeps)
- return _insert_extension_triggers(pkg, extensions, d_weakdeps)
+
+ total_deps = merge(weakdeps, deps)
+ return _insert_extension_triggers(pkg, extensions, total_deps)
  end
  end
  end
  end
  return nothing
 end
 
-function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any})
+function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, totaldeps::Dict{String, Any})
  for (ext, triggers) in extensions
  triggers = triggers::Union{String, Vector{String}}
  triggers isa String && (triggers = [triggers])
@@ -1451,7 +1458,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}
  push!(trigger1, gid)
  for trigger in triggers
  # TODO: Better error message if this lookup fails?
- uuid_trigger = UUID(weakdeps[trigger]::String)
+ uuid_trigger = UUID(totaldeps[trigger]::String)
  trigger_id = PkgId(uuid_trigger, trigger)
  if !haskey(explicit_loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
  trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id)

base/precompilation.jl

@@ -409,7 +409,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  all_extdeps_available = true
  for extdep_uuid in extdep_uuids
  extdep_name = env.names[extdep_uuid]
- if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name))
+ if extdep_uuid in keys(env.deps)
  push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name))
  else
  all_extdeps_available = false

doc/src/manual/code-loading.md

@@ -351,7 +351,7 @@ Since the primary environment is typically the environment of a project you're w
 
 ### [Package Extensions](@id man-extensions)
 
-A package "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of the project file. Those packages can have compat entries like other packages.
+A package "extension" is a module that is automatically loaded when a specified set of other packages (its "triggers") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The triggers of an extension are a subset of those packages listed under the `[weakdeps]` (and possibly, but uncommonly the `[deps]`) section of the project file. Those packages can have compat entries like other packages.
 
 ```toml
 name = "MyPackage"
@@ -371,27 +371,27 @@ FooExt = "ExtDep"
 ```
 
 The keys under `extensions` are the names of the extensions.
-They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded.
-If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity.
+They are loaded when all the packages on the right hand side (the triggers) of that extension are loaded.
+If an extension only has one trigger the list of triggers can be written as just a string for brevity.
 The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for
 extension `FooExt`.
 The content of an extension is often structured as:
 
 ```
 module FooExt
 
-# Load main package and extension dependencies
+# Load main package and triggers
 using MyPackage, ExtDep
 
-# Extend functionality in main package with types from the extension dependencies
+# Extend functionality in main package with types from the triggers
 MyPackage.func(x::ExtDep.SomeStruct) = ...
 
 end
 ```
 
 When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections
 are stored in the manifest file in the section for that package. The dependency lookup rules for
-a package are the same as for its "parent" except that the listed extension dependencies are also considered as
+a package are the same as for its "parent" except that the listed triggers are also considered as
 dependencies.
 
 ### [Workspaces](@id workspaces)

test/loading.jl

@@ -1057,7 +1057,9 @@ end
  $ew HasDepWithExtensions.do_something() || error("do_something errored")
  using ExtDep2
  $ew using ExtDep2
- $ew HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set")
+ using ExtDep3
+ $ew using ExtDep3
+ $ew HasExtensions.ext_dep_loaded || error("ext_dep_loaded not set")
  end
  """
  return `$(Base.julia_cmd()) $compile --startup-file=no -e $cmd`
@@ -1105,6 +1107,8 @@ end
  Base.get_extension(HasExtensions, :Extension) isa Module || error("expected extension to load")
  using ExtDep2
  Base.get_extension(HasExtensions, :ExtensionFolder) isa Module || error("expected extension to load")
+ using ExtDep3
+ Base.get_extension(HasExtensions, :ExtensionDep) isa Module || error("expected extension to load")
  end
  """
  for compile in (`--compiled-modules=no`, ``)

test/project/Extensions/EnvWithHasExtensions/Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.9.0-beta4"
+julia_version = "1.12.0-DEV"
 manifest_format = "2.0"
-project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef"
+project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f"
 
 [[deps.ExtDep]]
 deps = ["SomePackage"]
@@ -18,10 +18,12 @@ version = "0.1.0"
  [deps.HasExtensions.extensions]
  Extension = "ExtDep"
  ExtensionFolder = ["ExtDep", "ExtDep2"]
+ LinearAlgebraExt = "LinearAlgebra"
 
  [deps.HasExtensions.weakdeps]
  ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
  ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[deps.SomePackage]]
 path = "../SomePackage"

test/project/Extensions/ExtDep3.jl/Project.toml

@@ -0,0 +1,4 @@
+name = "ExtDep3"
+uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
+version = "0.1.0"
+authors = ["Kristoffer <kcarlsson89@gmail.com>"]

test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl

@@ -0,0 +1,5 @@
+module ExtDep3
+
+greet() = print("Hello World!")
+
+end # module ExtDep3

test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.0"
+julia_version = "1.12.0-DEV"
 manifest_format = "2.0"
-project_hash = "d523b3401f72a1ed34b7b43749fd2655c6b78542"
+project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0"
 
 [[deps.ExtDep]]
 deps = ["SomePackage"]
@@ -15,13 +15,20 @@ path = "../ExtDep2"
 uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
 version = "0.1.0"
 
+[[deps.ExtDep3]]
+path = "../ExtDep3.jl"
+uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
+version = "0.1.0"
+
 [[deps.HasExtensions]]
+deps = ["ExtDep3"]
 path = "../HasExtensions.jl"
 uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
 version = "0.1.0"
 
  [deps.HasExtensions.extensions]
  Extension = "ExtDep"
+ ExtensionDep = "ExtDep3"
  ExtensionFolder = ["ExtDep", "ExtDep2"]
  LinearAlgebraExt = "LinearAlgebra"
 

test/project/Extensions/HasDepWithExtensions.jl/Project.toml

@@ -5,4 +5,5 @@ version = "0.1.0"
 [deps]
 ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
 ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1"
 HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"

test/project/Extensions/HasExtensions.jl/Manifest.toml

@@ -1,7 +1,10 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.0-DEV"
+julia_version = "1.12.0-DEV"
 manifest_format = "2.0"
-project_hash = "c87947f1f1f070eea848950c304d668a112dec3d"
+project_hash = "c0bb526b75939a74a6195ee4819e598918a22ad7"
 
-[deps]
+[[deps.ExtDep3]]
+path = "../ExtDep3.jl"
+uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
+version = "0.1.0"

test/project/Extensions/HasExtensions.jl/Project.toml

@@ -2,12 +2,16 @@ name = "HasExtensions"
 uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
 version = "0.1.0"
 
+[deps]
+ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1"
+
 [weakdeps]
 ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
 ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [extensions]
 Extension = "ExtDep"
+ExtensionDep = "ExtDep3"
 ExtensionFolder = ["ExtDep", "ExtDep2"]
 LinearAlgebraExt = "LinearAlgebra"

test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl

@@ -0,0 +1,9 @@
+module ExtensionDep
+
+using HasExtensions, ExtDep3
+
+function __init__()
+ HasExtensions.ext_dep_loaded = true
+end
+
+end

test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl

@@ -6,5 +6,6 @@ foo(::HasExtensionsStruct) = 1
 
 ext_loaded = false
 ext_folder_loaded = false
+ext_dep_loaded = false
 
 end # module