Revert "Revert "prepare for sysimage excision""

src/Operations.jl

@@ -810,7 +810,12 @@ function download_source(ctx::Context; readonly=true)
     max_name = maximum(widths; init=0)
 
     # Check what registries the current pkg server tracks
-    server_registry_info = Registry.pkg_server_registry_info()
+    # Disable if precompiling to not access internet
+    server_registry_info = if Base.JLOptions().incremental == 0
+        Registry.pkg_server_registry_info()
+    else
+        nothing
+    end
 
     @sync begin
         jobs = Channel{eltype(pkgs_to_install)}(ctx.num_concurrent_downloads)

src/Pkg.jl

@@ -789,98 +789,6 @@ function _auto_precompile(ctx::Types.Context, pkgs::Vector{PackageSpec}=PackageS
     end
 end
 
-using LibGit2: LibGit2
-using Tar: Tar
-function _run_precompilation_script_setup()
-    tmp = mktempdir()
-    cd(tmp)
-    empty!(DEPOT_PATH)
-    pushfirst!(DEPOT_PATH, tmp)
-    touch("Project.toml")
-    Pkg.activate(".")
-    Pkg.generate("TestPkg")
-    uuid = TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"]
-    mv("TestPkg", "TestPkg.jl")
-    tree_hash = cd("TestPkg.jl") do
-        sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time()), 0)
-        repo = LibGit2.init(".")
-        LibGit2.add!(repo, "")
-        commit = LibGit2.commit(repo, "initial commit"; author=sig, committer=sig)
-        th = LibGit2.peel(LibGit2.GitTree, LibGit2.GitObject(repo, commit)) |> LibGit2.GitHash |> string
-        close(repo)
-        th
-    end
-    # Prevent cloning the General registry by adding a fake one
-    mkpath("registries/Registry/T/TestPkg")
-    write("registries/Registry/Registry.toml", """
-        name = "Registry"
-        uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
-        repo = "https://github.com/JuliaRegistries/Registry.git"
-        [packages]
-        $uuid = { name = "TestPkg", path = "T/TestPkg" }
-        """)
-    write("registries/Registry/T/TestPkg/Compat.toml", """
-          ["0"]
-          julia = "1"
-          """)
-    write("registries/Registry/T/TestPkg/Deps.toml", """
-          ["0"]
-          Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
-          """)
-    write("registries/Registry/T/TestPkg/Versions.toml", """
-          ["0.1.0"]
-          git-tree-sha1 = "$tree_hash"
-          """)
-    write("registries/Registry/T/TestPkg/Package.toml", """
-        name = "TestPkg"
-        uuid = "$uuid"
-        repo = "$(escape_string(tmp))/TestPkg.jl"
-        """)
-    Tar.create("registries/Registry", "registries/Registry.tar")
-    cmd = `$(Pkg.PlatformEngines.exe7z()) a "registries/Registry.tar.gz" -tgzip "registries/Registry.tar"`
-    run(pipeline(cmd, stdout = stdout_f(), stderr = stderr_f()))
-    write("registries/Registry.toml", """
-          git-tree-sha1 = "11b5fad51c4f98cfe0c145ceab0b8fb63fed6f81"
-          uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
-          path = "Registry.tar.gz"
-    """)
-    Base.rm("registries/Registry"; recursive=true)
-    return tmp
-end
-
-function _run_precompilation_script_artifact()
-    # Create simple artifact, bind it, then use it:
-    foo_hash = Pkg.Artifacts.create_artifact(dir -> touch(joinpath(dir, "foo")))
-    Artifacts.bind_artifact!("./Artifacts.toml", "foo", foo_hash)
-    # Also create multiple platform-specific ones because that's a codepath we need precompiled
-    Artifacts.bind_artifact!("./Artifacts.toml", "foo_plat", foo_hash; platform=Base.BinaryPlatforms.HostPlatform())
-    # Because @artifact_str doesn't work at REPL-level, we JIT out a file that we can include()
-    write("load_artifact.jl", """
-          Pkg.Artifacts.artifact"foo"
-          Pkg.Artifacts.artifact"foo_plat"
-          """)
-    foo_path = include("load_artifact.jl")
-end
-
-const CTRL_C = '\x03'
-const precompile_script = """
-    import Pkg
-    _pwd = pwd()
-    Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
-    tmp = Pkg._run_precompilation_script_setup()
-    $CTRL_C
-    Pkg.add("TestPkg")
-    Pkg.develop(Pkg.PackageSpec(path="TestPkg.jl"))
-    Pkg.add(Pkg.PackageSpec(path="TestPkg.jl/"))
-    Pkg.REPLMode.try_prompt_pkg_add(Symbol[:notapackage])
-    Pkg.update(; update_registry=false)
-    Pkg.precompile()
-    ] add Te\t\t$CTRL_C
-    ] st
-    $CTRL_C
-    Pkg._run_precompilation_script_artifact()
-    rm(tmp; recursive=true)
-    cd(_pwd)
-    """
+include("precompile.jl")
 
 end # module

src/precompile.jl

@@ -0,0 +1,122 @@
+using LibGit2: LibGit2
+using Tar: Tar
+using Downloads
+using REPL
+
+let
+function _run_precompilation_script_setup()
+    tmp = mktempdir()
+    cd(tmp)
+    empty!(DEPOT_PATH)
+    pushfirst!(DEPOT_PATH, tmp)
+    pushfirst!(LOAD_PATH, "@")
+    write("Project.toml",
+        """
+        name = "Hello"
+        uuid = "33cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+        """
+    )
+    mkdir("src")
+    write("src/Hello.jl",
+        """
+        module Hello
+        end
+        """
+    )
+    Pkg.activate(".")
+    Pkg.generate("TestPkg")
+    uuid = TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"]
+    mv("TestPkg", "TestPkg.jl")
+    tree_hash = cd("TestPkg.jl") do
+        sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time()), 0)
+        repo = LibGit2.init(".")
+        LibGit2.add!(repo, "")
+        commit = LibGit2.commit(repo, "initial commit"; author=sig, committer=sig)
+        th = LibGit2.peel(LibGit2.GitTree, LibGit2.GitObject(repo, commit)) |> LibGit2.GitHash |> string
+        close(repo)
+        th
+    end
+    # Prevent cloning the General registry by adding a fake one
+    mkpath("registries/Registry/T/TestPkg")
+    write("registries/Registry/Registry.toml", """
+        name = "Registry"
+        uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
+        repo = "https://github.com/JuliaRegistries/Registry.git"
+        [packages]
+        $uuid = { name = "TestPkg", path = "T/TestPkg" }
+        """)
+    write("registries/Registry/T/TestPkg/Compat.toml", """
+          ["0"]
+          julia = "1"
+          """)
+    write("registries/Registry/T/TestPkg/Deps.toml", """
+          ["0"]
+          Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+          """)
+    write("registries/Registry/T/TestPkg/Versions.toml", """
+          ["0.1.0"]
+          git-tree-sha1 = "$tree_hash"
+          """)
+    write("registries/Registry/T/TestPkg/Package.toml", """
+        name = "TestPkg"
+        uuid = "$uuid"
+        repo = "$(escape_string(tmp))/TestPkg.jl"
+        """)
+    Tar.create("registries/Registry", "registries/Registry.tar")
+    cmd = `$(Pkg.PlatformEngines.exe7z()) a "registries/Registry.tar.gz" -tgzip "registries/Registry.tar"`
+    run(pipeline(cmd, stdout = stdout_f(), stderr = stderr_f()))
+    write("registries/Registry.toml", """
+          git-tree-sha1 = "11b5fad51c4f98cfe0c145ceab0b8fb63fed6f81"
+          uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
+          path = "Registry.tar.gz"
+    """)
+    Base.rm("registries/Registry"; recursive=true)
+    return tmp
+end
+
+# SnoopPrecompile is useful but not available in Base
+# using SnoopPrecompile
+function pkg_precompile()
+    Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
+    # Default 30 sec grace period means we hang 30 seconds before precompiling finishes
+    redirect_stderr(devnull) do
+    redirect_stdout(devnull) do
+    Downloads.DOWNLOADER[] = Downloads.Downloader(; grace=1.0)
+    withenv("JULIA_PKG_SERVER" => nothing) do
+        # @precompile_setup begin
+            tmp = _run_precompilation_script_setup()
+            # @precompile_all_calls begin
+                withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0) do
+                    @show DEPOT_PATH
+                    @show LOAD_PATH
+
+                    Pkg.add("TestPkg")
+                    Pkg.develop(Pkg.PackageSpec(path="TestPkg.jl"))
+                    Pkg.add(Pkg.PackageSpec(path="TestPkg.jl/"))
+                    Pkg.REPLMode.try_prompt_pkg_add(Symbol[:notapackage])
+                    Pkg.update(; update_registry=false)
+                    Pkg.status()
+                end
+                Pkg.precompile()
+                try Base.rm(tmp; recursive=true)
+                catch
+                end
+
+                Base.precompile(Tuple{typeof(Pkg.REPLMode.promptf)})
+                Base.precompile(Tuple{typeof(Pkg.REPLMode.repl_init), REPL.LineEditREPL})
+                Base.precompile(Tuple{typeof(Pkg.API.status)})
+                Base.precompile(Tuple{typeof(Pkg.Types.read_project_compat), Base.Dict{String, Any}, Pkg.Types.Project})
+                Base.precompile(Tuple{typeof(Pkg.Versions.semver_interval), Base.RegexMatch})
+
+                Base.precompile(Tuple{typeof(REPL.LineEdit.complete_line), Pkg.REPLMode.PkgCompletionProvider, REPL.LineEdit.PromptState})
+                Base.precompile(Tuple{typeof(Pkg.REPLMode.complete_argument), Pkg.REPLMode.CommandSpec, Array{String, 1}, String, Int64, Int64})
+                Base.precompile(Tuple{typeof(Pkg.REPLMode.complete_add_dev), Base.Dict{Symbol, Any}, String, Int64, Int64})
+            # end
+        # end
+        end
+        end
+    end
+end
+
+pkg_precompile()
+end

test/api.jl

@@ -46,59 +46,6 @@ end
 include("FakeTerminals.jl")
 import .FakeTerminals.FakeTerminal
 
-@testset "Pkg.precompile_script" begin
-    function fake_repl(@nospecialize(f); options::REPL.Options=REPL.Options(confirm_exit=false))
-        # Use pipes so we can easily do blocking reads
-        # In the future if we want we can add a test that the right object
-        # gets displayed by intercepting the display
-        input = Pipe()
-        output = Pipe()
-        err = Pipe()
-        Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true)
-        Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true)
-        Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true)
-
-        repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true)
-        repl.options = options
-
-        f(input.in, output.out, repl)
-        t = @async begin
-            close(input.in)
-            close(output.in)
-            close(err.in)
-        end
-        @test read(err.out, String) == ""
-        #display(read(output.out, String))
-        Base.wait(t)
-        nothing
-    end
-    pwd_before = pwd()
-    fake_repl() do stdin_write, stdout_read, repl
-        repltask = @async REPL.run_repl(repl)
-
-        for line in split(Pkg.precompile_script, "\n"; keepempty=false)
-            sleep(0.1)
-            # Consume any extra output
-            if bytesavailable(stdout_read) > 0
-                copyback = readavailable(stdout_read)
-                #@info(copyback)
-            end
-
-            # Write the line
-            write(stdin_write, line, "\n")
-
-            # Read until some kind of prompt
-            readuntil(stdout_read, "\n")
-            readuntil(stdout_read, ">")
-            #@info(line)
-        end
-
-        write(stdin_write, "\x04")
-        wait(repltask)
-    end
-    cd(pwd_before) # something in the precompile_script changes the working directory
-end
-
 @testset "Pkg.precompile" begin
     # sequential precompile, depth-first
     isolate() do; cd_tempdir() do tmp