From f6f637adf6fa6d2c499c04020c06ae457d2e1d05 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 30 May 2023 15:09:40 -0400 Subject: [PATCH] cleanup: some fixes from effort to break up monolithic sysimg build (#49953) From #38119 --- base/Base.jl | 5 ++- base/initdefs.jl | 6 +-- base/sysimg.jl | 15 +++---- base/timing.jl | 9 ++-- contrib/generate_precompile.jl | 76 +++++++++++++++++++--------------- pkgimage.mk | 9 ++++ sysimage.mk | 8 +++- test/Makefile | 5 +++ test/cmdlineargs.jl | 7 ++-- test/loading.jl | 4 +- test/precompile.jl | 1 + 11 files changed, 88 insertions(+), 57 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index 65abe47f33d2d..8c5b8e13d3fb5 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -6,7 +6,7 @@ using Core.Intrinsics, Core.IR # to start, we're going to use a very simple definition of `include` # that doesn't require any function (except what we can get from the `Core` top-module) -const _included_files = Array{Tuple{Module,String},1}() +const _included_files = Array{Tuple{Module,String},1}(Core.undef, 1) function include(mod::Module, path::String) ccall(:jl_array_grow_end, Cvoid, (Any, UInt), _included_files, UInt(1)) Core.arrayset(true, _included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), arraylen(_included_files)) @@ -607,5 +607,8 @@ end end +# Ensure this file is also tracked +@assert !isassigned(_included_files, 1) +_included_files[1] = (parentmodule(Base), abspath(@__FILE__)) end # baremodule Base diff --git a/base/initdefs.jl b/base/initdefs.jl index 8a5f9e440a089..d1d37839a7c13 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -93,6 +93,7 @@ function append_default_depot_path!(DEPOT_PATH) path in DEPOT_PATH || push!(DEPOT_PATH, path) path = abspath(Sys.BINDIR, "..", "share", "julia") path in DEPOT_PATH || push!(DEPOT_PATH, path) + return DEPOT_PATH end function init_depot_path() @@ -111,6 +112,7 @@ function init_depot_path() else append_default_depot_path!(DEPOT_PATH) end + nothing end ## LOAD_PATH & ACTIVE_PROJECT ## @@ -220,9 +222,7 @@ function parse_load_path(str::String) end function init_load_path() - if Base.creating_sysimg - paths = ["@stdlib"] - elseif haskey(ENV, "JULIA_LOAD_PATH") + if haskey(ENV, "JULIA_LOAD_PATH") paths = parse_load_path(ENV["JULIA_LOAD_PATH"]) else paths = filter!(env -> env !== nothing, diff --git a/base/sysimg.jl b/base/sysimg.jl index b0eeffa5757ba..09ea015b0f903 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -9,11 +9,9 @@ using Base.MainInclude # ans, err, and sometimes Out import Base.MainInclude: eval, include # Ensure this file is also tracked -pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "Base.jl"))) -pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl"))) +pushfirst!(Base._included_files, (@__MODULE__, abspath(@__FILE__))) # set up depot & load paths to be able to find stdlib packages -@eval Base creating_sysimg = true Base.init_depot_path() Base.init_load_path() @@ -82,7 +80,7 @@ let m = Module() GC.@preserve m begin print_time = @eval m (mod, t) -> (print(rpad(string(mod) * " ", $maxlen + 3, "─")); - Base.time_print(t * 10^9); println()) + Base.time_print(stdout, t * 10^9); println()) print_time(Base, (Base.end_base_include - Base.start_base_include) * 10^(-9)) Base._track_dependencies[] = true @@ -104,7 +102,6 @@ let empty!(Core.ARGS) empty!(Base.ARGS) empty!(LOAD_PATH) - @eval Base creating_sysimg = false Base.init_load_path() # want to be able to find external packages in userimg.jl ccall(:jl_clear_implicit_imports, Cvoid, (Any,), Main) @@ -114,12 +111,12 @@ let tot_time = tot_time_base + tot_time_stdlib + tot_time_userimg println("Sysimage built. Summary:") - print("Base ──────── "); Base.time_print(tot_time_base * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_base / tot_time) * 100); println("%") - print("Stdlibs ───── "); Base.time_print(tot_time_stdlib * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_stdlib / tot_time) * 100); println("%") + print("Base ──────── "); Base.time_print(stdout, tot_time_base * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_base / tot_time) * 100); println("%") + print("Stdlibs ───── "); Base.time_print(stdout, tot_time_stdlib * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_stdlib / tot_time) * 100); println("%") if isfile("userimg.jl") - print("Userimg ───── "); Base.time_print(tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%") + print("Userimg ───── "); Base.time_print(stdout, tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%") end - print("Total ─────── "); Base.time_print(tot_time * 10^9); println(); + print("Total ─────── "); Base.time_print(stdout, tot_time * 10^9); println(); empty!(LOAD_PATH) empty!(DEPOT_PATH) diff --git a/base/timing.jl b/base/timing.jl index 3e1f3a3451149..7428fd36c6253 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -135,7 +135,7 @@ function format_bytes(bytes) # also used by InteractiveUtils end end -function time_print(elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, recompile_time=0, newline=false, _lpad=true) +function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, recompile_time=0, newline=false, _lpad=true) timestr = Ryu.writefixed(Float64(elapsedtime/1e9), 6) str = sprint() do io _lpad && print(io, length(timestr) < 10 ? (" "^(10 - length(timestr))) : "") @@ -169,8 +169,9 @@ function time_print(elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, re print(io, ": ", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% of which was recompilation") end parens && print(io, ")") + newline && print(io, "\n") end - newline ? println(str) : print(str) + print(io, str) nothing end @@ -178,7 +179,7 @@ function timev_print(elapsedtime, diff::GC_Diff, compile_times, _lpad) allocs = gc_alloc_count(diff) compile_time = first(compile_times) recompile_time = last(compile_times) - time_print(elapsedtime, diff.allocd, diff.total_time, allocs, compile_time, recompile_time, true, _lpad) + time_print(stdout, elapsedtime, diff.allocd, diff.total_time, allocs, compile_time, recompile_time, true, _lpad) padded_nonzero_print(elapsedtime, "elapsed time (ns)") padded_nonzero_print(diff.total_time, "gc time (ns)") padded_nonzero_print(diff.allocd, "bytes allocated") @@ -279,7 +280,7 @@ macro time(msg, ex) local _msg = $(esc(msg)) local has_msg = !isnothing(_msg) has_msg && print(_msg, ": ") - time_print(elapsedtime, diff.allocd, diff.total_time, gc_alloc_count(diff), first(compile_elapsedtimes), last(compile_elapsedtimes), true, !has_msg) + time_print(stdout, elapsedtime, diff.allocd, diff.total_time, gc_alloc_count(diff), first(compile_elapsedtimes), last(compile_elapsedtimes), true, !has_msg) val end end diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index e8901a7b462ea..8fa40e4920eea 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -1,13 +1,14 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# Prevent this from putting anyting into the Main namespace +@eval Module() begin + if Threads.maxthreadid() != 1 @warn "Running this file with multiple Julia threads may lead to a build error" Threads.maxthreadid() end if Base.isempty(Base.ARGS) || Base.ARGS[1] !== "0" Sys.__init_build() -# Prevent this from being put into the Main namespace -@eval Module() begin if !isdefined(Base, :uv_eventloop) Base.reinit_stdio() end @@ -234,6 +235,13 @@ end ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" +blackhole = Sys.isunix() ? "/dev/null" : "nul" +procenv = Dict{String,Any}( + "JULIA_HISTORY" => blackhole, + "JULIA_PROJECT" => nothing, # remove from environment + "JULIA_LOAD_PATH" => "@stdlib", + "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", + "TERM" => "") generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed start_time = time_ns() @@ -285,7 +293,9 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path))) $precompile_script """ - run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`) + p = run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg + --cpu-target=native --startup-file=no --color=yes`, procenv), + stdin=IOBuffer(s), stdout=debug_output)) n_step1 = 0 for f in (tmp_prec, tmp_proc) isfile(f) || continue @@ -305,23 +315,15 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Collect statements from running a REPL process and replaying our REPL script touch(precompile_file) pts, ptm = open_fake_pty() - blackhole = Sys.isunix() ? "/dev/null" : "nul" if have_repl - cmdargs = ```--color=yes - -e 'import REPL; REPL.Terminals.is_precompiling[] = true' - ``` + cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` else cmdargs = `-e nothing` end - p = withenv("JULIA_HISTORY" => blackhole, - "JULIA_PROJECT" => nothing, # remove from environment - "JULIA_LOAD_PATH" => Sys.iswindows() ? "@;@stdlib" : "@:@stdlib", - "JULIA_PKG_PRECOMPILE_AUTO" => "0", - "TERM" => "") do - run(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg - --cpu-target=native --startup-file=no -i $cmdargs```, - pts, pts, pts; wait=false) - end + p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg + --cpu-target=native --startup-file=no --color=yes -i $cmdargs```, procenv), + "JULIA_PKG_PRECOMPILE_AUTO" => "0"), + pts, pts, pts; wait=false) Base.close_stdio(pts) # Prepare a background process to copy output from process until `pts` is closed output_copy = Base.BufferStream() @@ -452,18 +454,16 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe failed = length(statements) - n_succeeded print_state("step3" => string("F$n_succeeded", failed > 0 ? " ($failed failed)" : "")) println() - if have_repl - # Seems like a reasonable number right now, adjust as needed - # comment out if debugging script - n_succeeded > 1500 || @warn "Only $n_succeeded precompile statements" - end + # Seems like a reasonable number right now, adjust as needed + # comment out if debugging script + n_succeeded > (have_repl ? 900 : 90) || @warn "Only $n_succeeded precompile statements" fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") fetch(step2) == :ok || throw("Step 2 of collecting precompiles failed.") tot_time = time_ns() - start_time println("Precompilation complete. Summary:") - print("Total ─────── "); Base.time_print(tot_time); println() + print("Total ─────── "); Base.time_print(stdout, tot_time); println() finally fancyprint && print(ansi_enablecursor) return @@ -474,22 +474,30 @@ generate_precompile_statements() # As a last step in system image generation, # remove some references to build time environment for a more reproducible build. Base.Filesystem.temp_cleanup_purge(force=true) -@eval Base PROGRAM_FILE = "" -@eval Sys begin - BINDIR = "" - STDLIB = "" -end -empty!(Base.ARGS) -empty!(Core.ARGS) -end # @eval -end # if +let stdout = Ref{IO}(stdout) + Base.PROGRAM_FILE = "" + Sys.BINDIR = "" + Sys.STDLIB = "" + empty!(Base.ARGS) + empty!(Core.ARGS) + empty!(Base.TOML_CACHE.d) + Base.TOML.reinit!(Base.TOML_CACHE.p, "") + + println("Outputting sysimage file...") + Base.stdout = Core.stdout + Base.stderr = Core.stderr -println("Outputting sysimage file...") -let pre_output_time = time_ns() # Print report after sysimage has been saved so all time spent can be captured + pre_output_time = time_ns() Base.postoutput() do output_time = time_ns() - pre_output_time - print("Output ────── "); Base.time_print(output_time); println() + let stdout = stdout[] + print(stdout, "Output ────── "); Base.time_print(stdout, output_time); println(stdout) + end + stdout[] = Core.stdout end end + +end # if +end # @eval diff --git a/pkgimage.mk b/pkgimage.mk index dcf9dd1303d47..0803a188851bb 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -5,7 +5,16 @@ include $(JULIAHOME)/Make.inc VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) +# set some influential environment variables export JULIA_DEPOT_PATH := $(build_prefix)/share/julia +export JULIA_LOAD_PATH := @stdlib +unexport JULIA_PROJECT := +unexport JULIA_BINDIR := + +default: release +release: all-release +debug: all-debug +all: release debug $(JULIA_DEPOT_PATH): mkdir -p $@ diff --git a/sysimage.mk b/sysimage.mk index 7ed61d471a153..e5bbfad119131 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -76,8 +76,12 @@ $(build_private_libdir)/sys.ji: $(build_private_libdir)/corecompiler.ji $(JULIAH define sysimg_builder $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sys.ji $$(JULIAHOME)/contrib/generate_precompile.jl @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ - if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ - JULIA_NUM_THREADS=1 \ + if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \ + WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ + JULIA_LOAD_PATH='@stdlib' \ + JULIA_PROJECT= \ + JULIA_DEPOT_PATH=':' \ + JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ diff --git a/test/Makefile b/test/Makefile index 24e137a5b1492..88dbe5b2b4ed6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -6,6 +6,11 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) STDLIBDIR := $(build_datarootdir)/julia/stdlib/$(VERSDIR) # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE +export JULIA_DEPOT_PATH := $(build_prefix)/share/julia +export JULIA_LOAD_PATH := @stdlib +unexport JULIA_PROJECT := +unexport JULIA_BINDIR := + TESTGROUPS = unicode strings compiler TESTS = all default stdlib $(TESTGROUPS) \ $(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \ diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 1d04926ef23af..7ebed56227d03 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -134,10 +134,11 @@ end let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` # tests for handling of ENV errors - let v = writereadpipeline("println(\"REPL: \", @which(less), @isdefined(InteractiveUtils))", - setenv(`$exename -i -E 'empty!(LOAD_PATH); @isdefined InteractiveUtils'`, + let v = writereadpipeline( + "println(\"REPL: \", @which(less), @isdefined(InteractiveUtils))", + setenv(`$exename -i -E '@assert isempty(LOAD_PATH); push!(LOAD_PATH, "@stdlib"); @isdefined InteractiveUtils'`, "JULIA_LOAD_PATH" => "", - "JULIA_DEPOT_PATH" => "", + "JULIA_DEPOT_PATH" => ";:", "HOME" => homedir())) @test v == ("false\nREPL: InteractiveUtilstrue\n", true) end diff --git a/test/loading.jl b/test/loading.jl index ea544c2635dbc..394c13c5f2962 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -692,7 +692,9 @@ mktempdir() do dir mkpath(vpath) script = "@assert startswith(Base.active_project(), $(repr(vpath)))" cmd = `$(Base.julia_cmd()) --startup-file=no -e $(script)` - cmd = addenv(cmd, "JULIA_DEPOT_PATH" => dir) + cmd = addenv(cmd, + "JULIA_DEPOT_PATH" => dir, + "JULIA_LOAD_PATH" => Sys.iswindows() ? ";" : ":") cmd = pipeline(cmd; stdout, stderr) @test success(cmd) end diff --git a/test/precompile.jl b/test/precompile.jl index de3510d49118d..9cb5a1d52d485 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -4,6 +4,7 @@ original_depot_path = copy(Base.DEPOT_PATH) original_load_path = copy(Base.LOAD_PATH) using Test, Distributed, Random +using REPL # doc lookup function Foo_module = :Foo4b3a94a1a081a8cb Foo2_module = :F2oo4b3a94a1a081a8cb