Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow Registry.{add,update}() to target specific depots #3088

Merged
merged 2 commits into from
May 20, 2022
+129 −103
Merged

Allow Registry.{add,update}() to target specific depots #3088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47bf5f9
Allow `Registry.{add,update}()` to target specific depots
staticfloat May 19, 2022
f77a4d7
Ensure update gets correct path
staticfloat May 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 104 additions & 103 deletions src/Registry/Registry.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ Pkg.Registry.add(RegistrySpec(url = "https://github.com/JuliaRegistries/General.
add(reg::Union{String,RegistrySpec}; kwargs...) = add([reg]; kwargs...)
add(regs::Vector{String}; kwargs...) = add(RegistrySpec[RegistrySpec(name = name) for name in regs]; kwargs...)
add(; kwargs...) = add(RegistrySpec[]; kwargs...)
function add(regs::Vector{RegistrySpec}; io::IO=stderr_f())
function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depot=depots1())
if isempty(regs)
download_default_registries(io, only_if_empty = false)
download_default_registries(io, only_if_empty = false; depot)
else
download_registries(io, regs)
download_registries(io, regs, depot)
end
end

Expand Down Expand Up @@ -95,20 +95,20 @@ end

pkg_server_url_hash(url::String) = Base.SHA1(split(url, '/')[end])

function download_default_registries(io::IO; only_if_empty::Bool = true)
function download_default_registries(io::IO; only_if_empty::Bool = true, depot=depots1())
installed_registries = reachable_registries()
# Only clone if there are no installed registries, unless called
# with false keyword argument.
if isempty(installed_registries) || !only_if_empty
printpkgstyle(io, :Installing, "known registries into $(pathrepr(depots1()))")
printpkgstyle(io, :Installing, "known registries into $(pathrepr(depot))")
registries = copy(DEFAULT_REGISTRIES)
for uuid in keys(pkg_server_registry_urls())
if !(uuid in (reg.uuid for reg in registries))
push!(registries, RegistrySpec(uuid = uuid))
end
end
filter!(reg -> !(reg.uuid in installed_registries), registries)
download_registries(io, registries)
download_registries(io, registries, depot)
return true
end
return false
Expand Down Expand Up @@ -346,119 +346,120 @@ Pkg.Registry.update(RegistrySpec(uuid = "23338594-aafe-5451-b93e-139f81909106"))
"""
update(reg::Union{String,RegistrySpec}; kwargs...) = update([reg]; kwargs...)
update(regs::Vector{String}; kwargs...) = update([RegistrySpec(name = name) for name in regs]; kwargs...)
function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true)
depot = depots1()
isempty(regs) && (regs = reachable_registries(; depots=depot))
regdir = joinpath(depot, "registries")
isdir(regdir) || mkpath(regdir)
# only allow one julia process to update registries at a time
FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
errors = Tuple{String, String}[]
registry_urls = pkg_server_registry_urls()
for reg in unique(r -> r.uuid, find_installed_registries(io, regs); seen=Set{UUID}())
let reg=reg, errors=errors
regpath = pathrepr(reg.path)
let regpath=regpath
if reg.tree_info !== nothing
printpkgstyle(io, :Updating, "registry at " * regpath)
old_hash = reg.tree_info
url = get(registry_urls, reg.uuid, nothing)
if url !== nothing
check_registry_state(reg)
end
if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
# TODO: update faster by using a diff, if available
# TODO: DRY with the code in `download_default_registries`
let new_hash = new_hash, url = url
if registry_read_from_tarball()
tmp = tempname()
try
download_verify(url, nothing, tmp)
catch err
push!(errors, (reg.path, "failed to download from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_read
end
# If we have an uncompressed Pkg server registry, remove it and get the compressed version
if isdir(reg.path)
Base.rm(reg.path; recursive=true, force=true)
end
registry_path = dirname(reg.path)
mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true)
hash = pkg_server_url_hash(url)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
open(joinpath(registry_path, reg.name * ".toml"), "w") do io
TOML.print(io, reg_info)
end
@label done_tarball_read
else
mktempdir() do tmp
function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true, depots = [depots1()])
for depot in depots
depot_regs = isempty(regs) ? reachable_registries(; depots=depot) : regs
regdir = joinpath(depot, "registries")
isdir(regdir) || mkpath(regdir)
# only allow one julia process to update registries in this depot at a time
FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
errors = Tuple{String, String}[]
registry_urls = pkg_server_registry_urls()
for reg in unique(r -> r.uuid, find_installed_registries(io, depot_regs; depots=[depot]); seen=Set{UUID}())
let reg=reg, errors=errors
regpath = pathrepr(reg.path)
let regpath=regpath
if reg.tree_info !== nothing
printpkgstyle(io, :Updating, "registry at " * regpath)
old_hash = reg.tree_info
url = get(registry_urls, reg.uuid, nothing)
if url !== nothing
check_registry_state(reg)
end
if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
# TODO: update faster by using a diff, if available
# TODO: DRY with the code in `download_default_registries`
let new_hash = new_hash, url = url
if registry_read_from_tarball()
tmp = tempname()
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
download_verify(url, nothing, tmp)
catch err
push!(errors, (reg.path, "failed to download and unpack from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_unpack
push!(errors, (reg.path, "failed to download from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_read
end
# If we have an uncompressed Pkg server registry, remove it and get the compressed version
if isdir(reg.path)
Base.rm(reg.path; recursive=true, force=true)
end
registry_path = dirname(reg.path)
mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true)
hash = pkg_server_url_hash(url)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
open(joinpath(registry_path, reg.name * ".toml"), "w") do io
TOML.print(io, reg_info)
end
@label done_tarball_read
else
mktempdir() do tmp
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
catch err
push!(errors, (reg.path, "failed to download and unpack from $(url). Exception: $(sprint(showerror, err))"))
@goto done_tarball_unpack
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
@label done_tarball_unpack
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
@label done_tarball_unpack
end
end
end
end
elseif isdir(joinpath(reg.path, ".git"))
printpkgstyle(io, :Updating, "registry at " * regpath)
LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
if LibGit2.isdirty(repo)
push!(errors, (regpath, "registry dirty"))
@goto done_git
end
if !LibGit2.isattached(repo)
push!(errors, (regpath, "registry detached"))
@goto done_git
end
if !("origin" in LibGit2.remotes(repo))
push!(errors, (regpath, "origin not in the list of remotes"))
@goto done_git
end
branch = LibGit2.headname(repo)
try
GitTools.fetch(io, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
catch e
e isa Pkg.Types.PkgError || rethrow()
push!(errors, (reg.path, "failed to fetch from repo: $(e.msg)"))
@goto done_git
end
ff_succeeded = try
LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
catch e
e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND || rethrow()
push!(errors, (reg.path, "branch origin/$branch not found"))
@goto done_git
end

if !ff_succeeded
try LibGit2.rebase!(repo, "origin/$branch")
elseif isdir(joinpath(reg.path, ".git"))
printpkgstyle(io, :Updating, "registry at " * regpath)
LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
if LibGit2.isdirty(repo)
push!(errors, (regpath, "registry dirty"))
@goto done_git
end
if !LibGit2.isattached(repo)
push!(errors, (regpath, "registry detached"))
@goto done_git
end
if !("origin" in LibGit2.remotes(repo))
push!(errors, (regpath, "origin not in the list of remotes"))
@goto done_git
end
branch = LibGit2.headname(repo)
try
GitTools.fetch(io, repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
catch e
e isa LibGit2.GitError || rethrow()
push!(errors, (reg.path, "registry failed to rebase on origin/$branch"))
e isa Pkg.Types.PkgError || rethrow()
push!(errors, (reg.path, "failed to fetch from repo: $(e.msg)"))
@goto done_git
end
ff_succeeded = try
LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
catch e
e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND || rethrow()
push!(errors, (reg.path, "branch origin/$branch not found"))
@goto done_git
end

if !ff_succeeded
try LibGit2.rebase!(repo, "origin/$branch")
catch e
e isa LibGit2.GitError || rethrow()
push!(errors, (reg.path, "registry failed to rebase on origin/$branch"))
@goto done_git
end
end
@label done_git
end
@label done_git
end
end
end
end
end
if !isempty(errors)
warn_str = "Some registries failed to update:"
for (reg, err) in errors
warn_str *= "\n — $reg — $err"
if !isempty(errors)
warn_str = "Some registries failed to update:"
for (reg, err) in errors
warn_str *= "\n — $reg — $err"
end
@error warn_str
end
@error warn_str
end # mkpidlock
end
end # mkpidlock
return
end

Expand Down
25 changes: 25 additions & 0 deletions test/registry.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,31 @@ end
@test isinstalled((name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a")))
end end

# Test Registry.add and Registry.update with explicit depot values
temp_pkg_dir() do depot_on_path; mktempdir() do depot_off_path
# No registries anywhere
@test isempty(Registry.reachable_registries())
@test isempty(Registry.reachable_registries(; depots=[depot_off_path]))

# After this, we have depots only in the depot that's off the path
Registry.add("General"; depot=depot_off_path)
@test isempty(Registry.reachable_registries())
@test length(Registry.reachable_registries(; depots=[depot_off_path])) == 1

# Test that `update()` with `depots` runs
io = Base.BufferStream()
Registry.update(; depots=[depot_off_path], io)
closewrite(io)
output = read(io, String)
@test occursin("registry at `$(depot_off_path)", output)

# Show that we can install `Example` off of that depot
empty!(Base.DEPOT_PATH)
push!(Base.DEPOT_PATH, depot_off_path)
Pkg.add("Example")
@test isinstalled((name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a")))
end end

# only clone default registry if there are no registries installed at all
temp_pkg_dir() do depot1; mktempdir() do depot2
append!(empty!(DEPOT_PATH), [depot1, depot2])
Expand Down