Skip to content

Commit

Permalink
more refactorings
Browse files Browse the repository at this point in the history
  • Loading branch information
@00vareladavid
00vareladavid committed Feb 20, 2020
1 parent 338b489 commit d69bdf8
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 52 deletions.
9 changes: 3 additions & 6 deletions src/API.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,14 +141,13 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=PR
stdlib_resolve!(pkgs)
for pkg in pkgs
if pkg.repo.source === nothing && pkg.version == VersionSpec() && pkg.name in keys(ctx.env.project.source)
pkg.repo.source = ctx.env.project.source[pkg.name]
pkg.repo.source = Operations.renormalize_source(Operations.projectdir(ctx),
ctx.env.project.source[pkg.name])
end
end

repo_pkgs = [pkg for pkg in pkgs if (pkg.repo.source !== nothing || pkg.repo.rev !== nothing)]
new = handle_repos_add!(ctx, repo_pkgs)
# repo + unpinned -> name, uuid, repo.rev, repo.source, tree_hash
# repo + pinned -> name, uuid, tree_hash

Types.update_registries(ctx)
registry_resolve!(ctx, pkgs)
Expand Down Expand Up @@ -225,10 +224,8 @@ function update(ctx::Context, pkgs::Vector{PackageSpec};
foreach(pkg -> pkg.mode = PKGMODE_PROJECT, pkgs)
project_deps_resolve!(ctx, pkgs)
ensure_resolved(ctx, pkgs)
uuids = UUID[pkg.uuid for pkg in pkgs]
pkgs = filter(pkg -> pkg.uuid in uuids, Operations.load_direct_deps(ctx, PackageSpec[], new))
pkgs = Operations.load_direct_deps(ctx, PackageSpec[], new; keep=Operations.uuids(pkgs))
end

Operations.update(ctx, pkgs, new, level)
return
end
Expand Down
10 changes: 9 additions & 1 deletion src/GitTools.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,15 @@ function clone(ctx, url, source_path; header=nothing, credentials=nothing, kwarg
end
end

function fetch(ctx, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, kwargs...)
function fetch(ctx, repo::LibGit2.GitRepo, remoteurl=nothing; force=false, header=nothing, credentials=nothing, kwargs...)
if !force
if LibGit2.path(repo) in ctx.updated_repos
return nothing
else
push!(ctx.updated_repos, LibGit2.path(repo))
end
end

if remoteurl === nothing
remoteurl = LibGit2.with(LibGit2.get(LibGit2.GitRemote, repo, "origin")) do remote
LibGit2.url(remote)
Expand Down
130 changes: 86 additions & 44 deletions src/Operations.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,22 +61,34 @@ function load_version(version, fixed, preserve::PreserveLevel)
end
end

function PackageSpec(uuid::UUID, entry::PackageEntry)
return PackageSpec(;
uuid = uuid,
name = entry.name,
path = entry.path,
repo = entry.repo,
pinned = entry.pinned,
tree_hash = entry.tree_hash,
version = something(entry.version, VersionSpec())
)
end

# TODO should preserve_none nuke the git revision?
function load_direct_deps(ctx::Context, pkgs::Vector{PackageSpec}=PackageSpec[], new::Vector{UUID}=UUID[];
download::Bool=true, preserve::PreserveLevel=PRESERVE_DIRECT)
unregistered::Bool=true, preserve::PreserveLevel=PRESERVE_DIRECT,
keep=nothing)
pkgs = copy(pkgs) # TODO this is needed for resolve tiers, but perhaps does not belong here
for (name::String, uuid::UUID) in ctx.env.project.deps
keep !== nothing && !in(uuid, keep) && continue
findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages
entry = manifest_info(ctx, uuid)
pkg = nothing
if entry === nothing
source = get(ctx.env.project.source, name, nothing)
if source !== nothing && download
if source !== nothing && unregistered
pkg = handle_unregistered_pkg(ctx, projectdir(ctx), source, new)
collect_unregistered!(ctx, pkg, pkgs, new)
else
# TODO do we have to normalize the source here?
# TODO I think we have to do relpath abspath trick
# TODO or maybe `handle_unregistered_pkg` will handle it downstream?
pkg = PackageSpec(; uuid = uuid, name = name, repo = GitRepo(; source = source))
end
else
Expand All @@ -89,6 +101,9 @@ function load_direct_deps(ctx::Context, pkgs::Vector{PackageSpec}=PackageSpec[],
tree_hash = entry.tree_hash,
version = load_version(entry.version, isfixed(entry), preserve),
)
if !is_tracking_registry(pkg) && unregistered
collect_unregistered_fixed!(ctx, pkg, pkgs)
end
end
push!(pkgs, pkg)
end
Expand Down Expand Up @@ -314,6 +329,7 @@ end
# adds any other packages which may be in the dependency graph
# all versioned packges should have a `tree_hash`
function resolve_versions!(ctx::Context, pkgs::Vector{PackageSpec})
check_registered(ctx, pkgs)
# compatibility
v = intersect(VERSION, project_compatibility(ctx, "julia"))
if isempty(v)
Expand Down Expand Up @@ -728,6 +744,15 @@ function download_source(ctx::Context, pkgs::Vector{PackageSpec},
return new_pkgs
end

function download(ctx::Context, pkgs::Vector{PackageSpec}, new::Vector{UUID}=UUID[];
platform::Platform=platform_key_abi())
union!(new, uuids(download_source(ctx, pkgs)))
# After downloading resolutionary packages, search for (Julia)Artifacts.toml files
# and ensure they are all downloaded and unpacked as well:
download_artifacts(ctx, pkgs; platform=platform)
return new
end

################################
# Manifest update and pruning #
################################
Expand Down Expand Up @@ -1043,7 +1068,6 @@ function targeted_resolve(ctx::Context, pkgs::Vector{PackageSpec}, preserve::Pre
elseif preserve == PRESERVE_NONE
pkgs = load_direct_deps(ctx, pkgs, new; preserve=preserve)
end
check_registered(ctx, pkgs)
resolve_versions!(ctx, pkgs)
return pkgs
end
Expand All @@ -1066,21 +1090,42 @@ function is_valid_repo(path)
return false
end

function handle_unregistered_pkg(ctx::Context, project_path::String, source::String, new::Vector{UUID})
function renormalize_source(project_path::String, source::String)
if !isurl(source)
use_abspath = isabspath(source)
source = project_rel_path(project_path, source)
source = use_abspath ? source : relpath(source, pwd())
end
return source
end

function handle_unregistered_pkg(ctx::Context, project_path::String, source::String, new::Vector{UUID};
force_add::Bool=false)
source = renormalize_source(project_path, source)
new_pkg = PackageSpec(; repo=GitRepo(;source=source))
if isurl(source) || is_valid_repo(source)
if force_add || isurl(source) || is_valid_repo(source)
Types.handle_repo_add!(ctx, new_pkg) && push!(new, new_pkg.uuid)
else
Types.handle_repo_develop!(ctx, new_pkg, false) && push!(new, new_pkg.uuid)
end
return new_pkg
end

function collect_unregistered_fixed!(ctx::Context, pkg::PackageSpec, pkgs::Vector{PackageSpec})
entry = manifest_info(ctx, pkg.uuid)
all_deps = load_all_deps
for (name, uuid) in entry.deps
entry = manifest_info(ctx, uuid)
entry === nothing && continue
new_pkg = PackageSpec(uuid, entry)
is_tracking_registry(new_pkg) && continue
findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages
push!(pkgs, new_pkg)
collect_unregistered_fixed!(ctx, new_pkg, pkgs)
end
return pkgs
end

function collect_unregistered!(ctx::Context, pkg::PackageSpec, pkgs::Vector{PackageSpec}, new::Vector{UUID})
project_path = project_rel_path(ctx, source_path(pkg))
projectfile = projectfile_path(project_path; strict=true)
Expand Down Expand Up @@ -1134,15 +1179,12 @@ end
function add(ctx::Context, pkgs::Vector{PackageSpec}, new=UUID[];
preserve::PreserveLevel=PRESERVE_TIERED, platform::Platform=platform_key_abi())
should_preserve_all = isempty(intersect(values(ctx.env.project.deps), uuids(pkgs)))
pkgs = map(pkg -> add_package(ctx, pkg, manifest_info(ctx, pkg.uuid)), pkgs)
foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) # update set of deps
pkgs = map(pkg -> add_package(ctx, pkg, manifest_info(ctx, pkg.uuid)), pkgs)
collect_unregistered!(ctx, pkgs, new)
pkgs = resolve(ctx, pkgs, new, preserve, should_preserve_all)
update_env!(ctx, pkgs)
union!(new, uuids(download_source(ctx, pkgs)))
# After downloading resolutionary packages, search for (Julia)Artifacts.toml files
# and ensure they are all downloaded and unpacked as well:
download_artifacts(ctx, pkgs; platform=platform)
new = download(ctx, pkgs; platform=platform)
show_update(ctx)
write_env(ctx.env) # write env before building
build_versions(ctx, new)
Expand All @@ -1152,13 +1194,11 @@ end
function develop(ctx::Context, pkgs::Vector{PackageSpec}, new::Vector{UUID};
preserve::PreserveLevel=PRESERVE_TIERED, platform::Platform=platform_key_abi())
should_preserve_all = isempty(intersect(values(ctx.env.project.deps), uuids(pkgs)))
# no need to look at manifest.. dev will just nuke whatever is there before
foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)
collect_unregistered!(ctx, pkgs, new)
pkgs = resolve(ctx, pkgs, new, preserve, should_preserve_all)
update_env!(ctx, pkgs)
union!(new, uuids(download_source(ctx, pkgs; readonly=true)))
download_artifacts(ctx, pkgs; platform=platform)
new = download(ctx, pkgs; platform=platform)
show_update(ctx)
write_env(ctx.env) # write env before building
build_versions(ctx, new)
Expand All @@ -1168,35 +1208,36 @@ function update_package!(ctx::Context, pkg::PackageSpec, level::UpgradeLevel, ne
if pkg.pinned || level == UPLEVEL_FIXED
return pkg
end
if is_tracking_repo(pkg) # repo packages have a version but are treated special
if !is_tracking_registry(pkg)
if level == UPLEVEL_MAJOR
Types.handle_repo_add!(ctx, pkg) && push!(new, pkg.uuid)
return pkg
else
return pkg
end
else
# TODO pkg.version should only by a number or an empty VersionSpec by this point
if pkg.version isa VersionNumber
r = level == UPLEVEL_PATCH ? VersionRange(pkg.version.major, pkg.version.minor) :
level == UPLEVEL_MINOR ? VersionRange(pkg.version.major) :
VersionRange()
pkg.version = VersionSpec(r)
source = renormalize_source(projectdir(ctx), something(pkg.path, pkg.repo.source))
new_pkg = PackageSpec(; repo=GitRepo(;source=source))
if is_tracking_path(pkg)
Types.handle_repo_develop!(ctx, new_pkg, false) && push!(new, new_pkg.uuid)
else
Types.handle_repo_add!(ctx, new_pkg) && push!(new, new_pkg.uuid)
end
return new_pkg
end
return pkg
end
# TODO pkg.version should only by a number or an empty VersionSpec by this point
if pkg.version isa VersionNumber
r = level == UPLEVEL_PATCH ? VersionRange(pkg.version.major, pkg.version.minor) :
level == UPLEVEL_MINOR ? VersionRange(pkg.version.major) :
VersionRange()
pkg.version = VersionSpec(r)
end
return pkg
end

function update(ctx::Context, pkgs::Vector{PackageSpec}, new::Vector{UUID}, level::UpgradeLevel)
pkgs = map(pkg -> update_package!(ctx, pkg, level, new), pkgs)
pkgs = load_direct_deps(ctx, pkgs, new; preserve = (level == UPLEVEL_FIXED ? PRESERVE_NONE : PRESERVE_DIRECT))
check_registered(ctx, pkgs)
collect_unregistered!(ctx, pkgs, new)
resolve_versions!(ctx, pkgs)
prune_manifest(ctx)
update_env!(ctx, pkgs)
union!(new, uuids(download_source(ctx, pkgs)))
download_artifacts(ctx, pkgs)
new = download(ctx, pkgs, new)
show_update(ctx)
write_env(ctx.env) # write env before building
build_versions(ctx, new)
Expand Down Expand Up @@ -1229,11 +1270,9 @@ end
function pin(ctx::Context, pkgs::Vector{PackageSpec})
foreach(pkg -> pin_package!(ctx, pkg, manifest_info(ctx, pkg.uuid)), pkgs)
pkgs = load_direct_deps(ctx, pkgs)
check_registered(ctx, pkgs)
resolve_versions!(ctx, pkgs)
update_env!(ctx, pkgs)
new = uuids(download_source(ctx, pkgs))
download_artifacts(ctx, pkgs)
new = download(ctx, pkgs)
show_update(ctx)
write_env(ctx.env) # write env before building
build_versions(ctx, new)
Expand Down Expand Up @@ -1266,15 +1305,12 @@ function free(ctx::Context, pkgs::Vector{PackageSpec})

if any(pkg -> pkg.version == VersionSpec(), pkgs)
pkgs = load_direct_deps(ctx, pkgs)
check_registered(ctx, pkgs)
resolve_versions!(ctx, pkgs)
update_env!(ctx, pkgs)
new = download_source(ctx, pkgs)
# TODO suspicious that this uses `new` instead of pkgs, is this intended?
download_artifacts(ctx, new)
new = download(ctx, pkgs)
show_update(ctx)
write_env(ctx.env) # write env before building
build_versions(ctx, uuids(new))
build_versions(ctx, new)
else
foreach(pkg -> manifest_info(ctx, pkg.uuid).pinned = false, pkgs)
show_update(ctx)
Expand Down Expand Up @@ -1371,7 +1407,7 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
sandbox_manifest = abspath!(sandbox_path, sandbox_env.manifest)
for (name, uuid) in sandbox_env.project.deps
entry = get(sandbox_manifest, uuid, nothing)
if entry !== nothing && isfixed(entry)
if entry !== nothing && !is_tracking_registry(entry)
subgraph = prune_manifest(sandbox_manifest, [uuid])
for (uuid, entry) in subgraph
if haskey(working_manifest, uuid)
Expand All @@ -1386,8 +1422,14 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
with_temp_env(tmp) do
temp_ctx = Context()
temp_ctx.env.project.deps[target.name] = target.uuid
temp_ctx.env.project.source[target.name] = target_path
for (uuid, entry) in temp_ctx.env.manifest
is_tracking_registry(entry) && continue
temp_ctx.env.project.source[entry.name] = something(entry.path, entry.repo.source)
end
write_env(temp_ctx.env, update_undo = false)
try
Pkg.status()
Pkg.resolve(; io=devnull)
@debug "Using _parent_ dep graph"
catch err# TODO
Expand Down Expand Up @@ -1607,11 +1649,11 @@ function diff_array(old_ctx::Union{Context,Nothing}, new_ctx::Context; manifest=
return idx === nothing ? nothing : pkgs[idx]
end

new = manifest ? load_manifest_deps(new_ctx) : load_direct_deps(new_ctx; download=false)
new = manifest ? load_manifest_deps(new_ctx) : load_direct_deps(new_ctx; unregistered=false)
if old_ctx === nothing
return [(pkg.uuid, nothing, pkg) for pkg in new]
end
old = manifest ? load_manifest_deps(old_ctx) : load_direct_deps(old_ctx; download=false)
old = manifest ? load_manifest_deps(old_ctx) : load_direct_deps(old_ctx; unregistered=false)

all_uuids = union([pkg.uuid for pkg in old], [pkg.uuid for pkg in new])
return [(uuid, index_pkgs(old, uuid), index_pkgs(new, uuid)) for uuid in all_uuids]
Expand Down
3 changes: 2 additions & 1 deletion src/Types.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ Base.@kwdef mutable struct Context
io::IO = something(DEFAULT_IO[], stderr)
use_libgit2_for_all_downloads::Bool = false
use_only_tarballs_for_downloads::Bool = false
updated_repos::Vector{String} = String[]
# NOTE: The JULIA_PKG_CONCURRENCY environment variable is likely to be removed in
# the future. It currently stands as an unofficial workaround for issue #795.
num_concurrent_downloads::Int = haskey(ENV, "JULIA_PKG_CONCURRENCY") ? parse(Int, ENV["JULIA_PKG_CONCURRENCY"]) : 8
Expand Down Expand Up @@ -1085,7 +1086,7 @@ function update_registries(ctx::Context, regs::Vector{RegistrySpec} = collect_re
end
branch = LibGit2.headname(repo)
try
GitTools.fetch(ctx, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
GitTools.fetch(ctx, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"], force=true)
catch e
e isa PkgError || rethrow()
push!(errors, (reg.path, "failed to fetch from repo"))
Expand Down
2 changes: 2 additions & 0 deletions test/new.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1683,6 +1683,8 @@ end
# free package tracking repo
isolate(loaded_depot=true) do
Pkg.add(Pkg.PackageSpec(; name="Example", rev="master"))
# also track a recursively unregistered dependency to make sure it is preserved
Pkg.add(Pkg.PackageSpec(; url="https://github.com/00vareladavid/UnregisteredA"))
Pkg.free("Example")
Pkg.dependencies(exuuid) do pkg
@test pkg.name == "Example"
Expand Down

0 comments on commit d69bdf8

Please sign in to comment.