Skip to content

Commit

Permalink
Merge pull request #3088 from JuliaLang/sf/add_update_depot_kwarg
Browse files Browse the repository at this point in the history 
Allow `Registry.{add,update}()` to target specific depots
  • Loading branch information
@staticfloat
staticfloat authored May 20, 2022
2 parents 951dca9 + f77a4d7 commit 610c768
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 103 deletions.
207 changes: 104 additions & 103 deletions src/Registry/Registry.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ Pkg.Registry.add(RegistrySpec(url = "https://github.com/JuliaRegistries/General.
add(reg::Union{String,RegistrySpec}; kwargs...) = add([reg]; kwargs...)
add(regs::Vector{String}; kwargs...) = add(RegistrySpec[RegistrySpec(name = name) for name in regs]; kwargs...)
add(; kwargs...) = add(RegistrySpec[]; kwargs...)
function add(regs::Vector{RegistrySpec}; io::IO=stderr_f())
function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depot=depots1())
if isempty(regs)
download_default_registries(io, only_if_empty = false)
download_default_registries(io, only_if_empty = false; depot)
else
download_registries(io, regs)
download_registries(io, regs, depot)
end
end

Expand Down Expand Up @@ -95,20 +95,20 @@ end

pkg_server_url_hash(url::String) = Base.SHA1(split(url, '/')[end])

function download_default_registries(io::IO; only_if_empty::Bool = true)
function download_default_registries(io::IO; only_if_empty::Bool = true, depot=depots1())
installed_registries = reachable_registries()
# Only clone if there are no installed registries, unless called
# with false keyword argument.
if isempty(installed_registries) || !only_if_empty
printpkgstyle(io, :Installing, "known registries into $(pathrepr(depots1()))")
printpkgstyle(io, :Installing, "known registries into $(pathrepr(depot))")
registries = copy(DEFAULT_REGISTRIES)
for uuid in keys(pkg_server_registry_urls())
if !(uuid in (reg.uuid for reg in registries))
push!(registries, RegistrySpec(uuid = uuid))
end
end
filter!(reg -> !(reg.uuid in installed_registries), registries)
download_registries(io, registries)
download_registries(io, registries, depot)
return true
end
return false
Expand Down Expand Up @@ -346,119 +346,120 @@ Pkg.Registry.update(RegistrySpec(uuid = "23338594-aafe-5451-b93e-139f81909106"))
"""
update(reg::Union{String,RegistrySpec}; kwargs...) = update([reg]; kwargs...)
update(regs::Vector{String}; kwargs...) = update([RegistrySpec(name = name) for name in regs]; kwargs...)
function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true)
depot = depots1()
isempty(regs) && (regs = reachable_registries(; depots=depot))
regdir = joinpath(depot, "registries")
isdir(regdir) || mkpath(regdir)
# only allow one julia process to update registries at a time
FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
errors = Tuple{String, String}[]
registry_urls = pkg_server_registry_urls()
for reg in unique(r -> r.uuid, find_installed_registries(io, regs); seen=Set{UUID}())
let reg=reg, errors=errors
regpath = pathrepr(reg.path)
let regpath=regpath
if reg.tree_info !== nothing
printpkgstyle(io, :Updating, "registry at " * regpath)
old_hash = reg.tree_info
url = get(registry_urls, reg.uuid, nothing)
if url !== nothing
check_registry_state(reg)
end
if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
# TODO: update faster by using a diff, if available
# TODO: DRY with the code in `download_default_registries`
let new_hash = new_hash, url = url
if registry_read_from_tarball()
tmp = tempname()
try
download_verify(url, nothing, tmp)
catch err
push!(errors, (reg.path, "failed to download from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_read
end
# If we have an uncompressed Pkg server registry, remove it and get the compressed version
if isdir(reg.path)
Base.rm(reg.path; recursive=true, force=true)
end
registry_path = dirname(reg.path)
mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true)
hash = pkg_server_url_hash(url)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
open(joinpath(registry_path, reg.name * ".toml"), "w") do io
TOML.print(io, reg_info)
end
@label done_tarball_read
else
mktempdir() do tmp
function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true, depots = [depots1()])
for depot in depots
depot_regs = isempty(regs) ? reachable_registries(; depots=depot) : regs
regdir = joinpath(depot, "registries")
isdir(regdir) || mkpath(regdir)
# only allow one julia process to update registries in this depot at a time
FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
errors = Tuple{String, String}[]
registry_urls = pkg_server_registry_urls()
for reg in unique(r -> r.uuid, find_installed_registries(io, depot_regs; depots=[depot]); seen=Set{UUID}())
let reg=reg, errors=errors
regpath = pathrepr(reg.path)
let regpath=regpath
if reg.tree_info !== nothing
printpkgstyle(io, :Updating, "registry at " * regpath)
old_hash = reg.tree_info
url = get(registry_urls, reg.uuid, nothing)
if url !== nothing
check_registry_state(reg)
end
if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
# TODO: update faster by using a diff, if available
# TODO: DRY with the code in `download_default_registries`
let new_hash = new_hash, url = url
if registry_read_from_tarball()
tmp = tempname()
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
download_verify(url, nothing, tmp)
catch err
push!(errors, (reg.path, "failed to download and unpack from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_unpack
push!(errors, (reg.path, "failed to download from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_read
end
# If we have an uncompressed Pkg server registry, remove it and get the compressed version
if isdir(reg.path)
Base.rm(reg.path; recursive=true, force=true)
end
registry_path = dirname(reg.path)
mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true)
hash = pkg_server_url_hash(url)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
open(joinpath(registry_path, reg.name * ".toml"), "w") do io
TOML.print(io, reg_info)
end
@label done_tarball_read
else
mktempdir() do tmp
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
catch err
push!(errors, (reg.path, "failed to download and unpack from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_unpack
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
@label done_tarball_unpack
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
@label done_tarball_unpack
end
end
end
end
elseif isdir(joinpath(reg.path, ".git"))
printpkgstyle(io, :Updating, "registry at " * regpath)
LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
if LibGit2.isdirty(repo)
push!(errors, (regpath, "registry dirty"))
@goto done_git
end
if !LibGit2.isattached(repo)
push!(errors, (regpath, "registry detached"))
@goto done_git
end
if !("origin" in LibGit2.remotes(repo))
push!(errors, (regpath, "origin not in the list of remotes"))
@goto done_git
end
branch = LibGit2.headname(repo)
try
GitTools.fetch(io, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
catch e
e isa Pkg.Types.PkgError || rethrow()
push!(errors, (reg.path, "failed to fetch from repo: $(e.msg)"))
@goto done_git
end
ff_succeeded = try
LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
catch e
e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND || rethrow()
push!(errors, (reg.path, "branch origin/$branch not found"))
@goto done_git
end

if !ff_succeeded
try LibGit2.rebase!(repo, "origin/$branch")
elseif isdir(joinpath(reg.path, ".git"))
printpkgstyle(io, :Updating, "registry at " * regpath)
LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
if LibGit2.isdirty(repo)
push!(errors, (regpath, "registry dirty"))
@goto done_git
end
if !LibGit2.isattached(repo)
push!(errors, (regpath, "registry detached"))
@goto done_git
end
if !("origin" in LibGit2.remotes(repo))
push!(errors, (regpath, "origin not in the list of remotes"))
@goto done_git
end
branch = LibGit2.headname(repo)
try
GitTools.fetch(io, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
catch e
e isa LibGit2.GitError || rethrow()
push!(errors, (reg.path, "registry failed to rebase on origin/$branch"))
e isa Pkg.Types.PkgError || rethrow()
push!(errors, (reg.path, "failed to fetch from repo: $(e.msg)"))
@goto done_git
end
ff_succeeded = try
LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
catch e
e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND || rethrow()
push!(errors, (reg.path, "branch origin/$branch not found"))
@goto done_git
end

if !ff_succeeded
try LibGit2.rebase!(repo, "origin/$branch")
catch e
e isa LibGit2.GitError || rethrow()
push!(errors, (reg.path, "registry failed to rebase on origin/$branch"))
@goto done_git
end
end
@label done_git
end
@label done_git
end
end
end
end
end
if !isempty(errors)
warn_str = "Some registries failed to update:"
for (reg, err) in errors
warn_str *= "\n — $reg — $err"
if !isempty(errors)
warn_str = "Some registries failed to update:"
for (reg, err) in errors
warn_str *= "\n — $reg — $err"
end
@error warn_str
end
@error warn_str
end # mkpidlock
end
end # mkpidlock
return
end

Expand Down
25 changes: 25 additions & 0 deletions test/registry.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,31 @@ end
@test isinstalled((name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a")))
end end

# Test Registry.add and Registry.update with explicit depot values
temp_pkg_dir() do depot_on_path; mktempdir() do depot_off_path
# No registries anywhere
@test isempty(Registry.reachable_registries())
@test isempty(Registry.reachable_registries(; depots=[depot_off_path]))

# After this, we have depots only in the depot that's off the path
Registry.add("General"; depot=depot_off_path)
@test isempty(Registry.reachable_registries())
@test length(Registry.reachable_registries(; depots=[depot_off_path])) == 1

# Test that `update()` with `depots` runs
io = Base.BufferStream()
Registry.update(; depots=[depot_off_path], io)
closewrite(io)
output = read(io, String)
@test occursin("registry at `$(depot_off_path)", output)

# Show that we can install `Example` off of that depot
empty!(Base.DEPOT_PATH)
push!(Base.DEPOT_PATH, depot_off_path)
Pkg.add("Example")
@test isinstalled((name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a")))
end end

# only clone default registry if there are no registries installed at all
temp_pkg_dir() do depot1; mktempdir() do depot2
append!(empty!(DEPOT_PATH), [depot1, depot2])
Expand Down

0 comments on commit 610c768

Please sign in to comment.