diff --git a/Makefile b/Makefile index c31517c4f1e3a..d41f3e50d9752 100644 --- a/Makefile +++ b/Makefile @@ -262,13 +262,21 @@ ifeq ($(OS),WINNT) -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ ifeq ($(JULIA_BUILD_MODE),release) -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ else ifeq ($(JULIA_BUILD_MODE),debug) -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it. -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/ -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ + # The rest are compiler dependencies, as an example memcpy is exported by msvcrt + # These are files from mingw32 and required for creating shared libraries like our caches. + -$(INSTALL_M) $(build_libdir)/libgcc_s.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libgcc.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libmsvcrt.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin @@ -333,6 +341,11 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif + # Cache stdlibs + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no $(JULIAHOME)/contrib/cache_stdlibs.jl) + # CI uses `--check-bounds=yes` which impacts the cache flags + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes $(JULIAHOME)/contrib/cache_stdlibs.jl) + # Copy in all .jl sources as well mkdir -p $(DESTDIR)$(datarootdir)/julia/base $(DESTDIR)$(datarootdir)/julia/test cp -R -L $(JULIAHOME)/base/* $(DESTDIR)$(datarootdir)/julia/base diff --git a/base/Base.jl b/base/Base.jl index 07b0a2f0f0d49..22e8b48288efc 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -418,6 +418,7 @@ include("threadcall.jl") include("uuid.jl") include("pkgid.jl") include("toml_parser.jl") +include("linking.jl") include("loading.jl") # misc useful functions & macros diff --git a/base/Makefile b/base/Makefile index c320465a10492..d92302b766988 100644 --- a/base/Makefile +++ b/base/Makefile @@ -81,6 +81,10 @@ ifeq ($(DARWIN_FRAMEWORK), 1) @echo "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ else @echo "const DARWIN_FRAMEWORK = false" >> $@ +endif +ifeq ($(OS), Darwin) + @echo "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ + @echo "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ endif @echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ diff --git a/base/linking.jl b/base/linking.jl new file mode 100644 index 0000000000000..288279347f1c5 --- /dev/null +++ b/base/linking.jl @@ -0,0 +1,147 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license +module Linking + +import Base.Libc: Libdl + +# inlined LLD_jll +# These get calculated in __init__() +const PATH = Ref("") +const LIBPATH = Ref("") +const PATH_list = String[] +const LIBPATH_list = String[] +const lld_path = Ref{String}() +const lld_exe = Sys.iswindows() ? "lld.exe" : "lld" + +if Sys.iswindows() + const LIBPATH_env = "PATH" + const LIBPATH_default = "" + const pathsep = ';' +elseif Sys.isapple() + const LIBPATH_env = "DYLD_FALLBACK_LIBRARY_PATH" + const LIBPATH_default = "~/lib:/usr/local/lib:/lib:/usr/lib" + const pathsep = ':' +else + const LIBPATH_env = "LD_LIBRARY_PATH" + const LIBPATH_default = "" + const pathsep = ':' +end + +function adjust_ENV!(env::Dict, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) + if adjust_LIBPATH + LIBPATH_base = get(env, LIBPATH_env, expanduser(LIBPATH_default)) + if !isempty(LIBPATH_base) + env[LIBPATH_env] = string(LIBPATH, pathsep, LIBPATH_base) + else + env[LIBPATH_env] = LIBPATH + end + end + if adjust_PATH && (LIBPATH_env != "PATH" || !adjust_LIBPATH) + if !isempty(get(env, "PATH", "")) + env["PATH"] = string(PATH, pathsep, env["PATH"]) + else + env["PATH"] = PATH + end + end + return env +end + +function __init_lld_path() + # Prefer our own bundled lld, but if we don't have one, pick it up off of the PATH + # If this is an in-tree build, `lld` will live in `tools`. Otherwise, it'll be in `libexec` + for bundled_lld_path in (joinpath(Sys.BINDIR, Base.LIBEXECDIR, lld_exe), + joinpath(Sys.BINDIR, "..", "tools", lld_exe), + joinpath(Sys.BINDIR, lld_exe)) + if isfile(bundled_lld_path) + lld_path[] = abspath(bundled_lld_path) + return + end + end + lld_path[] = something(Sys.which(lld_exe), lld_exe) + return +end + +const VERBOSE = Ref{Bool}(false) + +function __init__() + VERBOSE[] = parse(Bool, get(ENV, "JULIA_VERBOSE_LINKING", "false")) + + __init_lld_path() + PATH[] = dirname(lld_path[]) + if Sys.iswindows() + # On windows, the dynamic libraries (.dll) are in Sys.BINDIR ("usr\\bin") + append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), Sys.BINDIR]) + else + append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), abspath(Sys.BINDIR, Base.LIBDIR)]) + end + LIBPATH[] = join(LIBPATH_list, pathsep) + return +end + +function lld(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) + env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) + return Cmd(Cmd([lld_path[]]); env) +end + +function ld() + default_args = `` + @static if Sys.iswindows() + # LLD supports mingw style linking + flavor = "gnu" + m = Sys.ARCH == :x86_64 ? "i386pep" : "i386pe" + default_args = `-m $m -Bdynamic --enable-auto-image-base --allow-multiple-definition` + elseif Sys.isapple() + flavor = "darwin" + arch = Sys.ARCH == :aarch64 ? :arm64 : Sys.ARCH + default_args = `-arch $arch -undefined dynamic_lookup -platform_version macos $(Base.MACOS_PRODUCT_VERSION) $(Base.MACOS_PLATFORM_VERSION)` + else + flavor = "gnu" + end + + `$(lld()) -flavor $flavor $default_args` +end + +const WHOLE_ARCHIVE = if Sys.isapple() + "-all_load" +else + "--whole-archive" +end + +const NO_WHOLE_ARCHIVE = if Sys.isapple() + "" +else + "--no-whole-archive" +end + +const SHARED = if Sys.isapple() + "-dylib" +else + "-shared" +end + +is_debug() = ccall(:jl_is_debugbuild, Cint, ()) == 1 +libdir() = abspath(Sys.BINDIR, Base.LIBDIR) +private_libdir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) +if Sys.iswindows() + shlibdir() = Sys.BINDIR +else + shlibdir() = libdir() +end + +function link_image_cmd(path, out) + LIBDIR = "-L$(libdir())" + PRIVATE_LIBDIR = "-L$(private_libdir())" + SHLIBDIR = "-L$(shlibdir())" + LIBS = is_debug() ? ("-ljulia-debug", "-ljulia-internal-debug") : ("-ljulia", "-ljulia-internal") + @static if Sys.iswindows() + LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") + end + + V = VERBOSE[] ? "--verbose" : "" + `$(ld()) $V $SHARED -o $out $WHOLE_ARCHIVE $path $NO_WHOLE_ARCHIVE $LIBDIR $PRIVATE_LIBDIR $SHLIBDIR $LIBS` +end + +function link_image(path, out, internal_stderr::IO = stderr, internal_stdout::IO = stdout) + run(link_image_cmd(path, out), Base.DevNull(), stderr, stdout) +end + +end # module Linking diff --git a/base/loading.jl b/base/loading.jl index 2ce769405022a..06ba6e733b6d7 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -908,7 +908,8 @@ function find_all_in_cache_path(pkg::PkgId) isdir(path) || continue for file in readdir(path, sort = false) # no sort given we sort later if !((pkg.uuid === nothing && file == entryfile * ".ji") || - (pkg.uuid !== nothing && startswith(file, entryfile * "_"))) + (pkg.uuid !== nothing && startswith(file, entryfile * "_") && + endswith(file, ".ji"))) continue end filepath = joinpath(path, file) @@ -925,13 +926,15 @@ function find_all_in_cache_path(pkg::PkgId) end end +ocachefile_from_cachefile(cachefile) = string(chopsuffix(cachefile, ".ji"), ".", Base.Libc.dlext) + # use an Int counter so that nested @time_imports calls all remain open const TIMING_IMPORTS = Threads.Atomic{Int}(0) # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any}) +function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}) assert_havelock(require_lock) timing_imports = TIMING_IMPORTS[] > 0 try @@ -941,36 +944,18 @@ function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any} t_comp_before = cumulative_compile_time_ns() end - @debug "Loading cache file $path for $pkg" - sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), path, depmods, false) + if ocachepath !== nothing + @debug "Loading object cache file $ocachepath for $pkg" + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint), ocachepath, depmods, false) + else + @debug "Loading cache file $path for $pkg" + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), path, depmods, false) + end if isa(sv, Exception) return sv end - sv = sv::SimpleVector - restored = sv[1]::Vector{Any} - for M in restored - M = M::Module - if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing - push!(Base.Docs.modules, M) - end - if parentmodule(M) === M - register_root_module(M) - end - end - - # Register this cache path now - If Requires.jl is loaded, Revise may end - # up looking at the cache path during the init callback. - get!(PkgOrigin, pkgorigins, pkg).cachepath = path - inits = sv[2]::Vector{Any} - if !isempty(inits) - unlock(require_lock) # temporarily _unlock_ during these callbacks - try - ccall(:jl_init_restored_modules, Cvoid, (Any,), inits) - finally - lock(require_lock) - end - end + restored = register_restored_modules(sv, pkg, path) for M in restored M = M::Module @@ -1005,6 +990,35 @@ function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any} end end +function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String) + # This function is also used by PkgCacheInspector.jl + restored = sv[1]::Vector{Any} + for M in restored + M = M::Module + if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing + push!(Base.Docs.modules, M) + end + if parentmodule(M) === M + register_root_module(M) + end + end + + # Register this cache path now - If Requires.jl is loaded, Revise may end + # up looking at the cache path during the init callback. + get!(PkgOrigin, pkgorigins, pkg).cachepath = path + + inits = sv[2]::Vector{Any} + if !isempty(inits) + unlock(require_lock) # temporarily _unlock_ during these callbacks + try + ccall(:jl_init_restored_modules, Cvoid, (Any,), inits) + finally + lock(require_lock) + end + end + return restored +end + function run_package_callbacks(modkey::PkgId) assert_havelock(require_lock) unlock(require_lock) @@ -1207,7 +1221,7 @@ end # loads a precompile cache file, ignoring stale_cachefile tests # assuming all depmods are already loaded and everything is valid -function _tryrequire_from_serialized(modkey::PkgId, path::String, sourcepath::String, depmods::Vector{Any}) +function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any}) assert_havelock(require_lock) loaded = nothing if root_module_exists(modkey) @@ -1229,7 +1243,7 @@ function _tryrequire_from_serialized(modkey::PkgId, path::String, sourcepath::St package_locks[modkey] = Threads.Condition(require_lock) try set_pkgorigin_version_path(modkey, sourcepath) - loaded = _include_from_serialized(modkey, path, depmods) + loaded = _include_from_serialized(modkey, path, ocachepath, depmods) finally loading = pop!(package_locks, modkey) notify(loading, loaded, all=true) @@ -1248,13 +1262,23 @@ end # loads a precompile cache file, ignoring stale_cachefile tests # load the best available (non-stale) version of all dependent modules first -function _tryrequire_from_serialized(pkg::PkgId, path::String) +function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames io = open(path, "r") try iszero(isvalid_cache_header(io)) && return ArgumentError("Invalid header in cache file $path.") - depmodnames = parse_cache_header(io)[3] + _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io) + pkgimage = !isempty(clone_targets) + if pkgimage + ocachepath !== nothing || return ArgumentError("Expected ocachepath to be provided") + isfile(ocachepath) || return ArgumentError("Ocachepath $ocachpath is not a file.") + ocachepath == ocachefile_from_cachefile(path) || return ArgumentError("$ocachepath is not the expected ocachefile") + # TODO: Check for valid clone_targets? + isvalid_pkgimage_crc(io, ocachepath) || return ArgumentError("Invalid checksum in cache file $ocachepath.") + else + @assert ocachepath === nothing + end isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.") finally close(io) @@ -1270,7 +1294,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String) depmods[i] = dep end # then load the file - return _include_from_serialized(pkg, path, depmods) + return _include_from_serialized(pkg, path, ocachepath, depmods) end # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it @@ -1278,12 +1302,13 @@ end @constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128) assert_havelock(require_lock) paths = find_all_in_cache_path(pkg) + ocachefile = nothing for path_to_try in paths::Vector{String} staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try) if staledeps === true continue end - staledeps = staledeps::Vector{Any} + staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} # finish checking staledeps module graph for i in 1:length(staledeps) dep = staledeps[i] @@ -1296,8 +1321,8 @@ end if modstaledeps === true continue end - modstaledeps = modstaledeps::Vector{Any} - staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps) + modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} + staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath) modfound = true break end @@ -1308,6 +1333,7 @@ end end end if staledeps === true + ocachefile = nothing continue end try @@ -1319,19 +1345,20 @@ end for i in 1:length(staledeps) dep = staledeps[i] dep isa Module && continue - modpath, modkey, modpath_to_try, modstaledeps = dep::Tuple{String, PkgId, String, Vector{Any}} - dep = _tryrequire_from_serialized(modkey, modpath_to_try, modpath, modstaledeps) + modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}} + dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps) if !isa(dep, Module) - @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modpath." exception=dep + @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep staledeps = true break end staledeps[i] = dep end if staledeps === true + ocachefile = nothing continue end - restored = _include_from_serialized(pkg, path_to_try, staledeps) + restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) if !isa(restored, Module) @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored else @@ -1640,7 +1667,8 @@ function _require(pkg::PkgId, env=nothing) end # fall-through to loading the file locally else - m = _tryrequire_from_serialized(pkg, cachefile) + cachefile, ocachefile = cachefile::Tuple{String, Union{Nothing, String}} + m = _tryrequire_from_serialized(pkg, cachefile, ocachefile) if !isa(m, Module) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else @@ -1676,10 +1704,11 @@ function _require(pkg::PkgId, env=nothing) return loaded end -function _require_from_serialized(uuidkey::PkgId, path::String) +# Only used from test/precompile.jl +function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}) @lock require_lock begin set_pkgorigin_version_path(uuidkey, nothing) - newm = _tryrequire_from_serialized(uuidkey, path) + newm = _tryrequire_from_serialized(uuidkey, path, ocachepath) newm isa Module || throw(newm) insert_extension_triggers(uuidkey) # After successfully loading, notify downstream consumers @@ -1879,9 +1908,11 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end const PRECOMPILE_TRACE_COMPILE = Ref{String}() -function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) +function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, + concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists + output_o === nothing || rm(output_o, force=true) depot_path = map(abspath, DEPOT_PATH) dl_load_path = map(abspath, DL_LOAD_PATH) load_path = map(abspath, Base.load_path()) @@ -1900,11 +1931,20 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d for (pkg, build_id) in concrete_deps push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") end + + if output_o !== nothing + cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) + opt_level = Base.JLOptions().opt_level + opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes` + else + cpu_target = nothing + opts = `-O0 --output-ji $(output) --output-incremental=yes` + end + deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) deps = deps_eltype * "[" * join(deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` - io = open(pipeline(addenv(`$(julia_cmd()::Cmd) -O0 - --output-ji $output --output-incremental=yes + io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) --startup-file=no --history-file=no --warn-overwrite=yes --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") $trace @@ -1937,6 +1977,14 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64)::String crc = _crc32c(something(Base.active_project(), "")) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) + crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) + + cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) + if cpu_target === nothing + cpu_target = unsafe_string(JLOptions().cpu_target) + end + crc = _crc32c(cpu_target, crc) + crc = _crc32c(prefs_hash, crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) @@ -1983,44 +2031,92 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # create a temporary file in `cachepath` directory, write the cache in it, # write the checksum, _and then_ atomically move the file to `cachefile`. mkpath(cachepath) + cache_objects = JLOptions().use_pkgimages != 0 tmppath, tmpio = mktemp(cachepath) + + if cache_objects + tmppath_o, tmpio_o = mktemp(cachepath) + tmppath_so, tmpio_so = mktemp(cachepath) + else + tmppath_o = nothing + end local p try close(tmpio) - p = create_expr_cache(pkg, path, tmppath, concrete_deps, internal_stderr, internal_stdout) + if cache_objects + close(tmpio_o) + close(tmpio_so) + end + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout) + if success(p) + if cache_objects + # Run linker over tmppath_o + Linking.link_image(tmppath_o, tmppath_so) + end + + # Read preferences hash back from .ji file (we can't precompute because + # we don't actually know what the list of compile-time preferences are without compiling) + prefs_hash = preferences_hash(tmppath) + cachefile = compilecache_path(pkg, prefs_hash) + ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing + + # append checksum for so to the end of the .ji file: + crc_so = UInt32(0) + if cache_objects + crc_so = open(_crc32c, tmppath_so, "r") + end + # append extra crc to the end of the .ji file: open(tmppath, "r+") do f if iszero(isvalid_cache_header(f)) error("Invalid header for $pkg in new cache file $(repr(tmppath)).") end + seekend(f) + write(f, crc_so) seekstart(f) write(f, _crc32c(f)) end + # inherit permission from the source file (and make them writable) chmod(tmppath, filemode(path) & 0o777 | 0o200) - - # Read preferences hash back from .ji file (we can't precompute because - # we don't actually know what the list of compile-time preferences are without compiling) - prefs_hash = preferences_hash(tmppath) - cachefile = compilecache_path(pkg, prefs_hash) + if cache_objects + # Ensure that the user can execute the `.so` we're generating + # Note that on windows, `filemode(path)` typically returns `0o666`, so this + # addition of the execute bit for the user is doubly needed. + chmod(tmppath_so, filemode(path) & 0o777 | 0o300) + end # prune the directory with cache files if pkg.uuid !== nothing entrypath, entryfile = cache_file_entry(pkg) - cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath)) + cachefiles = filter!(x -> startswith(x, entryfile * "_") && endswith(x, ".ji"), readdir(cachepath)) + if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[] idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2] - rm(joinpath(cachepath, cachefiles[idx]); force=true) + evicted_cachefile = joinpath(cachepath, cachefiles[idx]) + @debug "Evicting file from cache" evicted_cachefile + rm(evicted_cachefile; force=true) + try + rm(ocachefile_from_cachefile(evicted_cachefile); force=true) + catch + end end end # this is atomic according to POSIX (not Win32): rename(tmppath, cachefile; force=true) - return cachefile + if cache_objects + rename(tmppath_so, ocachefile::String; force=true) + end + return cachefile, ocachefile end finally rm(tmppath, force=true) + if cache_objects + rm(tmppath_o, force=true) + rm(tmppath_so, force=true) + end end if p.exitcode == 125 return PrecompilableError() @@ -2034,9 +2130,26 @@ function module_build_id(m::Module) return (UInt128(hi) << 64) | lo end -isvalid_cache_header(f::IOStream) = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid},), f.ios) # returns checksum id or zero +function isvalid_cache_header(f::IOStream) + pkgimage = Ref{UInt8}() + checksum = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid}, Ptr{UInt8}, Ptr{Int64}, Ptr{Int64}), f.ios, pkgimage, Ref{Int64}(), Ref{Int64}()) # returns checksum id or zero + + if !iszero(checksum) && pkgimage[] != 0 + @debug "Cache header was for pkgimage" + return UInt64(0) # We somehow read the header for a pkgimage and not a ji + end + return checksum +end isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) +function isvalid_pkgimage_crc(f::IOStream, ocachefile::String) + seekstart(f) # TODO necessary + seek(f, filesize(f) - 8) + expected_crc_so = read(f, UInt32) + crc_so = open(_crc32c, ocachefile, "r") + expected_crc_so == crc_so +end + struct CacheHeaderIncludes id::PkgId filename::String @@ -2045,6 +2158,7 @@ struct CacheHeaderIncludes end function parse_cache_header(f::IO) + flags = read(f, UInt8) modules = Vector{Pair{PkgId, UInt64}}() while true n = read(f, Int32) @@ -2118,7 +2232,10 @@ function parse_cache_header(f::IO) build_id |= read(f, UInt64) push!(required_modules, PkgId(uuid, sym) => build_id) end - return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash + l = read(f, Int32) + clone_targets = read(f, l) + + return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags end function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) @@ -2140,8 +2257,6 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) end end - - preferences_hash(f::IO) = parse_cache_header(f)[6] function preferences_hash(cachefile::String) io = open(cachefile, "r") @@ -2155,7 +2270,6 @@ function preferences_hash(cachefile::String) end end - function cache_dependencies(f::IO) _, (includes, _), modules, _... = parse_cache_header(f) return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime @@ -2382,6 +2496,15 @@ get_compiletime_preferences(uuid::UUID) = collect(get(Vector{String}, COMPILETIM get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid) get_compiletime_preferences(::Nothing) = String[] +function check_clone_targets(clone_targets) + try + ccall(:jl_check_pkgimage_clones, Cvoid, (Ptr{Cchar},), clone_targets) + return true + catch + return false + end +end + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false) @@ -2395,10 +2518,33 @@ end @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" return true # invalid cache file end - modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io) + modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io) if isempty(modules) return true # ignore empty file end + if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 + @debug "Rejecting cache file $cachefile for $modkey since the flags are mismatched" cachefile_flags=flags current_flags=ccall(:jl_cache_flags, UInt8, ()) + return true + end + pkgimage = !isempty(clone_targets) + if pkgimage + ocachefile = ocachefile_from_cachefile(cachefile) + if JLOptions().use_pkgimages == 0 + # presence of clone_targets means native code cache + @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage" + return true + end + if !check_clone_targets(clone_targets) + @debug "Rejecting cache file $cachefile for $modkey since pkgimage can't be loaded on this target" + return true + end + if !isfile(ocachefile) + @debug "Rejecting cache file $cachefile for $modkey since pkgimage $ocachefile was not found" + return true + end + else + ocachefile = nothing + end id = first(modules) if id.first != modkey && modkey != PkgId("") @debug "Rejecting cache file $cachefile for $modkey since it is for $id instead" @@ -2461,7 +2607,7 @@ end # now check if this file is fresh relative to its source files if !skip_timecheck - if !samefile(includes[1].filename, modpath) + if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" return true # cache file was compiled from a different path end @@ -2474,6 +2620,16 @@ end end for chi in includes f, ftime_req = chi.filename, chi.mtime + if !isfile(f) + _f = fixup_stdlib_path(f) + if isfile(_f) && startswith(_f, Sys.STDLIB) + # mtime is changed by extraction + @debug "Skipping mtime check for file $f used by $cachefile, since it is a stdlib" + continue + end + @debug "Rejecting stale cache file $cachefile because file $f does not exist" + return true + end ftime = mtime(f) is_stale = ( ftime != ftime_req ) && ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes @@ -2493,13 +2649,20 @@ end return true end + if pkgimage + if !isvalid_pkgimage_crc(io, ocachefile::String) + @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum" + return true + end + end + curr_prefs_hash = get_preferences_hash(id.uuid, prefs) if prefs_hash != curr_prefs_hash @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" return true end - return depmods # fresh cachefile + return depmods, ocachefile # fresh cachefile finally close(io) end @@ -2572,4 +2735,5 @@ end precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) -precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), IO, IO)) +precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO)) +precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO)) diff --git a/base/options.jl b/base/options.jl index 48a8f7ff59d38..dda0e8b377076 100644 --- a/base/options.jl +++ b/base/options.jl @@ -38,6 +38,7 @@ struct JLOptions handle_signals::Int8 use_sysimage_native_code::Int8 use_compiled_modules::Int8 + use_pkgimages::Int8 bindto::Ptr{UInt8} outputbc::Ptr{UInt8} outputunoptbc::Ptr{UInt8} diff --git a/base/util.jl b/base/util.jl index 877446ab1887f..8d1a4a9fa02ef 100644 --- a/base/util.jl +++ b/base/util.jl @@ -133,7 +133,7 @@ See also [`print`](@ref), [`println`](@ref), [`show`](@ref). printstyled(stdout, msg...; bold=bold, underline=underline, blink=blink, reverse=reverse, hidden=hidden, color=color) """ - Base.julia_cmd(juliapath=joinpath(Sys.BINDIR, julia_exename())) + Base.julia_cmd(juliapath=joinpath(Sys.BINDIR, julia_exename()); cpu_target) Return a julia command similar to the one of the running process. Propagates any of the `--cpu-target`, `--sysimage`, `--compile`, `--sysimage-native-code`, @@ -148,10 +148,15 @@ Among others, `--math-mode`, `--warn-overwrite`, and `--trace-compile` are notab !!! compat "Julia 1.5" The flags `--color` and `--startup-file` were added in Julia 1.5. + +!!! compat "Julia 1.9" + The keyword argument `cpu_target` was added. """ -function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename())) +function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Union{Nothing,String} = nothing) opts = JLOptions() - cpu_target = unsafe_string(opts.cpu_target) + if cpu_target === nothing + cpu_target = unsafe_string(opts.cpu_target) + end image_file = unsafe_string(opts.image_file) addflags = String[] let compile = if opts.compile_enabled == 0 @@ -220,6 +225,12 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename())) if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end + if opts.use_pkgimages == 0 + push!(addflags, "--pkgimages=no") + else + # If pkgimage is set, malloc_log and code_coverage should not + @assert opts.malloc_log == 0 && opts.code_coverage == 0 + end return `$julia -C$cpu_target -J$image_file $addflags` end @@ -487,10 +498,17 @@ function _crc32c(io::IO, nb::Integer, crc::UInt32=0x00000000) end _crc32c(io::IO, crc::UInt32=0x00000000) = _crc32c(io, typemax(Int64), crc) _crc32c(io::IOStream, crc::UInt32=0x00000000) = _crc32c(io, filesize(io)-position(io), crc) -_crc32c(uuid::UUID, crc::UInt32=0x00000000) = - ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, uuid.value, 16) +_crc32c(uuid::UUID, crc::UInt32=0x00000000) = _crc32c(uuid.value, crc) +_crc32c(x::UInt128, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, x, 16) _crc32c(x::UInt64, crc::UInt32=0x00000000) = ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt64}, Csize_t), crc, x, 8) +_crc32c(x::UInt32, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt32}, Csize_t), crc, x, 4) +_crc32c(x::UInt16, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt16}, Csize_t), crc, x, 2) +_crc32c(x::UInt8, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt8}, Csize_t), crc, x, 1) """ @kwdef typedef @@ -655,7 +673,8 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), seed !== nothing && push!(tests, "--seed=0x$(string(seed % UInt128, base=16))") # cast to UInt128 to avoid a minus sign ENV2 = copy(ENV) ENV2["JULIA_CPU_THREADS"] = "$ncores" - ENV2["JULIA_DEPOT_PATH"] = mktempdir(; cleanup = true) + pathsep = Sys.iswindows() ? ";" : ":" + ENV2["JULIA_DEPOT_PATH"] = string(mktempdir(; cleanup = true), pathsep) # make sure the default depots can be loaded delete!(ENV2, "JULIA_LOAD_PATH") delete!(ENV2, "JULIA_PROJECT") try diff --git a/contrib/cache_stdlibs.jl b/contrib/cache_stdlibs.jl new file mode 100644 index 0000000000000..bdcc3d9535fa4 --- /dev/null +++ b/contrib/cache_stdlibs.jl @@ -0,0 +1,49 @@ +# Stdlibs sorted in dependency, then alphabetical, order by contrib/print_sorted_stdlibs.jl +# Run with the `--exclude-sysimage` option to filter out all packages included in the system image +stdlibs = [ + # No dependencies + + # 1-depth packages + :GMP_jll, + :LLVMLibUnwind_jll, + :LibUV_jll, + :LibUnwind_jll, + :MbedTLS_jll, + :OpenLibm_jll, + :PCRE2_jll, + :Zlib_jll, + :dSFMT_jll, + :libLLVM_jll, + + # 2-depth packages + :LibSSH2_jll, + :MPFR_jll, + + # 3-depth packages + :LibGit2_jll, + + # 7-depth packages + :LLD_jll, + :SuiteSparse_jll, + + # 9-depth packages + :Statistics, + :SuiteSparse, +] + +depot = abspath(Sys.BINDIR, "..", "share", "julia") + +if haskey(ENV, "JULIA_CPU_TARGET") + target = ENV["JULIA_CPU_TARGET"] +else + target = "native" +end + +@info "Caching stdlibrary to" depot target +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, depot) + +for pkg in stdlibs + pkgid = Base.identify_package(string(pkg)) + Base.compilecache(pkgid) +end diff --git a/contrib/print_sorted_stdlibs.jl b/contrib/print_sorted_stdlibs.jl index 28d75f079b9dd..6bc2023c4f1cc 100644 --- a/contrib/print_sorted_stdlibs.jl +++ b/contrib/print_sorted_stdlibs.jl @@ -12,11 +12,12 @@ function check_flag(flag) end if check_flag("--help") || check_flag("-h") - println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls]") + println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage]") end # Allow users to ask for JLL or no JLLs exclude_jlls = check_flag("--exclude-jlls") +exclude_sysimage = check_flag("--exclude-sysimage") # Default to the `stdlib/vX.Y` directory STDLIB_DIR = get(ARGS, 1, joinpath(@__DIR__, "..", "usr", "share", "julia", "stdlib")) @@ -80,12 +81,20 @@ if exclude_jlls filter!(p -> !endswith(p, "_jll"), sorted_projects) end +if exclude_sysimage + loaded_modules = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) + filter!(p->!in(p, loaded_modules), sorted_projects) +end + # Print out sorted projects, ready to be pasted into `sysimg.jl` last_depth = 0 println(" # Stdlibs sorted in dependency, then alphabetical, order by contrib/print_sorted_stdlibs.jl") if exclude_jlls println(" # Run with the `--exclude-jlls` option to filter out all JLL packages") end +if exclude_sysimage + println(" # Run with the `--exclude-sysimage` option to filter out all packages included in the system image") +end println(" stdlibs = [") println(" # No dependencies") for p in sorted_projects diff --git a/deps/checksums/compilersupportlibraries b/deps/checksums/compilersupportlibraries index 721ad2e8a8759..098c181ca5c87 100644 --- a/deps/checksums/compilersupportlibraries +++ b/deps/checksums/compilersupportlibraries @@ -1,92 +1,92 @@ -CompilerSupportLibraries.v1.0.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 -CompilerSupportLibraries.v1.0.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/3908fa1a2f739b330e787468c9bfb5c8 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/1741e3403ac7aa99e7cfd9a01222c4153ed300f47cc1b347e1af1a6cd07a82caaa54b9cfbebae8751440420551621cc6524504413446d104f9493dff2c081853 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/2444dbb7637b32cf543675cc12330878 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/8537f0b243df8544350c884021b21c585fd302e8dd462a30a6ee84c7a36a049133262e5d1bc362f972066b8e8d6a091c32c3b746bab1feb9fccf2e7cca65756c -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/d79c1434594c0c5e7d6be798bf52c99e -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/7e71accc401a45b51b298702fb4c79a2fc856c7b28f0935f6ad3a0db5381c55fe5432daff371842930d718024b7c6c1d80e2bd09d397145203673bebbe3496ae -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/f212059053d99558a9b0bf54b20180e1 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/5c104b1282cec8a944e5d008f44a4d60f4394fd5d797fec7d1f487d13e7328cd9c88ec4916dabf18596d87160756bda914e4f8c5a356b5577f9349d0d9e976d6 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/3e3b3795ee93ef317223050e803a9875 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/85d3c955e15f66bfe8bfec2f28c9160bc03d4d531ea4ffe6bc6b51e0d69ccea3ab67a16ca752dabc870861c407381c4519d75c6be3832e8dccd6122ec8c6ed75 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/cf2d1315f6a348af2e6c065e2a286e7a -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/58420377bc77aa7678034ee5f708eb6be7db359faef2c2638869765453633da9bf455512bd88e95b38ae0428ecc4053561517b176b2371129bdaef9d8d5dadfd -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran3.tar.gz/md5/6decf8fd5afb50451771c761e63a8917 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/4984724bcc847724b1bc005b6f760a18b68147f7d5402d0faf4e28fc0d14fa10975368a951f9caf2a8856500046dec8343043274557d58269e77492b929a9e4b -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran4.tar.gz/md5/39d1e8a3baa144c018d3eaf7f3806482 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/fc4d429279c5a93b6c28b6e911b1e7cfd1c1cfe46f11f2e901b3832ce90d45f49d3d29f0ef18518a94af6cc8651f67c4ed81672680f9281ada390440b172a2af -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran5.tar.gz/md5/37dabd9cd224c9fed9633dedccb6c565 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/b253149e72eef9486888fbaace66e9b6945f4477f6b818f64f3047331165b0e2bc17aa6e3fc8c88686a72e478eb62c8f53883415d5419db448d8016fa3a1da5e -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran3.tar.gz/md5/afdd32bfadd465848e6be458817a44ae -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran3.tar.gz/sha512/eebd679c499143014514c7c9d1875dedbbab9e3af51526c4dd445a9e3dbade95d24522da8bbad0a50ab400755e47b018828b324c4ad7705e212ccd990e34439a -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran4.tar.gz/md5/bc4a0f0b7cea328f7e8850583774496b -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran4.tar.gz/sha512/82285b67946212b49cddf6259f2c60ff5469f8c5263ccefe44f1d93ace98ab68e2c152e1b54434b2f075fd8d192c06d5451bc8cca26d951ad15f3453102f02b5 -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran5.tar.gz/md5/177f0232abce8d523882530ed7a93092 -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran5.tar.gz/sha512/db80acf0f2434f28ee7680e1beb34f564940071815d1ad89fb5913cbd9ac24da528e826d0d54be6265a7340ebd661b6d308ed79d96b67fa5d8c98dc3f1bee8d6 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/c723e7d3c3038f59b9bf0cc3a65826bc -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/0545561ccd7e525b6cd86935366a2724a5e013411a1c01564db21b66da5fef959cf06b0839b96f1dc2c970eb6c8fb19c012e6cd2c17bc381b55420c72fe1b9f6 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/763bd82645d2f3c72b6244d68bebb40f -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/700e719eeab486915a9fb0954125cb9a3e9a813d7a069eca05be3a16621f4875668918a5ed5f645e734ac62b0c2ddbaa6234adc9109e98fb88b8ca1197481ed8 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/18e90d15dc6dd0a836e9aa076b342105 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/9ff61e8da2b431a8cb09818bde5daab2d7b8cf7a934f184f14ea50eccf5796ae91558e06a22137eb021c4055c54faf4a524a54dbbd718e8ea0abb5dcec844fdb -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/4e5e4b23dc87450738da33926a07511d -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/fc09879d94b750e75775d8b64a41ab9924d675fb53c5700467604412928fe7f5cb21911da0f64898d2463fa77ffbaf4c96c397b9060f4746eec152747930cddc -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/9a92138ed69aa317a932a615c6e62d69 -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/0b7785379936a2a209b074177b1424dd7e00b29b5165f564e799b0aa4e06a582e9d616525d97274ba2507cb88192028f1ac485d3f99bdc7ee53fc63c1a7e85de -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/8ffee3d6de5197c7a1f354d72c8238fa -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/deadc4d7224c84f9b82dc956b69e815c44ae036802838365d870ab9f58c8bcf8ce0645f2f387c8ff344ac2108fc8e7e1ee907fa55e93c91aa5d9fd921bf3fdcb -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/87449e72e3f33dbb69b7053cdc2649d4 -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/5ce02ad10c6f4686a476eb2a5de2988cd8b482f5e693db2880c84ad1c82f468ef03fe01b9d0feefe5d4ee741d1d16643d36b144e6261ed32311b3b6f312fac2f -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/0407cde92cfa42fa89ac83217ca0ec16 -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/032c831f1166a336551138939ac40eb2c68a048ce786c0c1403b879a20c1b706caac16d22560b2c7f2b3d6373986c347188675674116005ca251336ee048d09f -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/23418763b808371ee94772a90d501f4d -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/7867b843551457b11bda7821dd384c1c1cf23b80a308b2058a693de7b7da099f0b37eb0a6de2b84c04b625a68c60eea55138e200d5d6ec6f6af09bd7ce406a96 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/e3d33ae03c18affea74699bdc1fabb68 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/42013f4921de5a69ad857195ce5c19ad1bca3c920d79699e5501f1f4534ab132fabd422362b2b5056f5d182215d6c069db5df460bafa700903faf962cc00f77b -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/d40c1e8c0393213c6057c53a12f44175 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/fe7baa4de7490065ab7b953cc12f41462a24bcb49d0a4a64b23249e98e7569b19bb1cb455af2f76090e34066a7d3cdd7a48cae6515ce6c7a5c8486b0cacc5106 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/48541b90f715c4c86ee4da0570275947 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/7f2683fb98e80f12629f4ed3bea9fd59d32b7e7a9ed1699e782d8e238ff0915ecc61bf00adaf4597cfe41caf82cdca0f9be250f595f5f0bea6d8f77dba99eaf4 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/4547059eb905995667be48bf85d49911 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/7400fdabc924434ab4a4949248c3603887ac06ffd2f205ae33e14495d86cd4f816bbd1999eeafa0257f518df1e7f7c522f596e847a71dbfbfccff4859f50acc7 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/46267543cad6584d7b7b9fcc8f18f21d -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/0353d7d724be48d4185d3c181692970b7996f53f6a01723072aa5c94b53a8c5055faeed30df51659c252a46f4b941dec0cb24569323e3c85c166f14c5b7c8e9e -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/14dba2897a6e9d370fa9091c045375fc -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/10b79f9c059839f5b57fa8d2a381a034c4067262c4088bd354d14ea56bec097878069383aa9cfadaa09d73bd20fc348fb61662d863a8d62cb25d7af6b8e29858 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/eed836d1addeb10d0901f836724aff1e -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/e33eca424d1529a1fb23ba9cf7fac345ed1cfc8073c975b6b31ca44d2e8c3f5083af65433df009b22483dceb2e43149f3c1e8433681fec5fb812e1d5b4243ce4 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/d5ae9f9519341fdaabf62267c89461d2 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/6421aa5d1bd6f08ad43f59ed4dc1bef8b9b598ebbbd3e48149730f3bec3471f8e2c02ffb338427326924290b8f52ef9e626e3313448bc931a61d866c5dc544ae -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/fc1df521395362a5aaa2e2aeef707207 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/f2e5a08e3cae171242ae6a20d2d4838c1529ce042745dc466148b7bbc06896d94476fd05c7787e6e8641bea752dfc0e6b09e95b160bede600d20d2ad68e7705f -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/80c337837a9032e4c9614f0d3218993b -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/cf07e459ca55cb9ee3d38e6858320530c1d1ab2ffd35bfa2a33b2505d3189f13b9743a0e279d70f85d227cee8a8974448f1371a122dcbea03fb1e414f8df8337 -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/792cae36932dd53af20b7f61c80f623b -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/805f2b64fe9d2b94fc6c966945e10458d8d1c47a8d95fcda057c03a13999d7d0f136c754e4b1e152faaf23e4949861c2ad42b4437dba19f59b3db745d7a76108 -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/063c07fcbba4b9c3bd23ab0d987f1dbb -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/1d0344b30b5fb34a63f6844be0501c0ad08f1116b0c7b00e13d47860cc6bbdd39734416ad3b492414a28ba1744240bd05aca0d1560873f687d3f61747058626b +CompilerSupportLibraries.v1.0.2+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 +CompilerSupportLibraries.v1.0.2+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/3908fa1a2f739b330e787468c9bfb5c8 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/1741e3403ac7aa99e7cfd9a01222c4153ed300f47cc1b347e1af1a6cd07a82caaa54b9cfbebae8751440420551621cc6524504413446d104f9493dff2c081853 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/2444dbb7637b32cf543675cc12330878 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/8537f0b243df8544350c884021b21c585fd302e8dd462a30a6ee84c7a36a049133262e5d1bc362f972066b8e8d6a091c32c3b746bab1feb9fccf2e7cca65756c +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/d79c1434594c0c5e7d6be798bf52c99e +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/7e71accc401a45b51b298702fb4c79a2fc856c7b28f0935f6ad3a0db5381c55fe5432daff371842930d718024b7c6c1d80e2bd09d397145203673bebbe3496ae +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/f212059053d99558a9b0bf54b20180e1 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/5c104b1282cec8a944e5d008f44a4d60f4394fd5d797fec7d1f487d13e7328cd9c88ec4916dabf18596d87160756bda914e4f8c5a356b5577f9349d0d9e976d6 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/3e3b3795ee93ef317223050e803a9875 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/85d3c955e15f66bfe8bfec2f28c9160bc03d4d531ea4ffe6bc6b51e0d69ccea3ab67a16ca752dabc870861c407381c4519d75c6be3832e8dccd6122ec8c6ed75 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/cf2d1315f6a348af2e6c065e2a286e7a +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/58420377bc77aa7678034ee5f708eb6be7db359faef2c2638869765453633da9bf455512bd88e95b38ae0428ecc4053561517b176b2371129bdaef9d8d5dadfd +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran3.tar.gz/md5/6decf8fd5afb50451771c761e63a8917 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/4984724bcc847724b1bc005b6f760a18b68147f7d5402d0faf4e28fc0d14fa10975368a951f9caf2a8856500046dec8343043274557d58269e77492b929a9e4b +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran4.tar.gz/md5/39d1e8a3baa144c018d3eaf7f3806482 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/fc4d429279c5a93b6c28b6e911b1e7cfd1c1cfe46f11f2e901b3832ce90d45f49d3d29f0ef18518a94af6cc8651f67c4ed81672680f9281ada390440b172a2af +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran5.tar.gz/md5/37dabd9cd224c9fed9633dedccb6c565 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/b253149e72eef9486888fbaace66e9b6945f4477f6b818f64f3047331165b0e2bc17aa6e3fc8c88686a72e478eb62c8f53883415d5419db448d8016fa3a1da5e +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran3.tar.gz/md5/afdd32bfadd465848e6be458817a44ae +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran3.tar.gz/sha512/eebd679c499143014514c7c9d1875dedbbab9e3af51526c4dd445a9e3dbade95d24522da8bbad0a50ab400755e47b018828b324c4ad7705e212ccd990e34439a +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran4.tar.gz/md5/bc4a0f0b7cea328f7e8850583774496b +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran4.tar.gz/sha512/82285b67946212b49cddf6259f2c60ff5469f8c5263ccefe44f1d93ace98ab68e2c152e1b54434b2f075fd8d192c06d5451bc8cca26d951ad15f3453102f02b5 +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran5.tar.gz/md5/177f0232abce8d523882530ed7a93092 +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran5.tar.gz/sha512/db80acf0f2434f28ee7680e1beb34f564940071815d1ad89fb5913cbd9ac24da528e826d0d54be6265a7340ebd661b6d308ed79d96b67fa5d8c98dc3f1bee8d6 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/756718e5eaa4547b874a71a8e3545492 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/c21c1be10ca8810f56e435b3629e2ab0678926ea9c4f4c3dd003f9e292c075493b83df04401d3bcf7738f1a44098f674f9b01bba9db4b9a9e45ad7af3497444e +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/65ce0024bf8fe3276addbf185ed03e48 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/5e8105a12ab04e2949e41eda50a060dea04ccd98660c7528cfc86e120fe61cca8bab878fd2c92a3858f02ac3f3c55d0e48789907e5fbd2392a8e84b183ed4636 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/b7727324d550f637209db795238c46a4 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/864b1db2642e68665b9d3322563c7ce964835d0e720325ea00b193e2cbf6791760e0014710e2a79876165ab0daffa6d53d61b87a5034f956ba6e255b0144652c +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/4e5e4b23dc87450738da33926a07511d +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/fc09879d94b750e75775d8b64a41ab9924d675fb53c5700467604412928fe7f5cb21911da0f64898d2463fa77ffbaf4c96c397b9060f4746eec152747930cddc +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/9a92138ed69aa317a932a615c6e62d69 +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/0b7785379936a2a209b074177b1424dd7e00b29b5165f564e799b0aa4e06a582e9d616525d97274ba2507cb88192028f1ac485d3f99bdc7ee53fc63c1a7e85de +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/8ffee3d6de5197c7a1f354d72c8238fa +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/deadc4d7224c84f9b82dc956b69e815c44ae036802838365d870ab9f58c8bcf8ce0645f2f387c8ff344ac2108fc8e7e1ee907fa55e93c91aa5d9fd921bf3fdcb +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/87449e72e3f33dbb69b7053cdc2649d4 +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/5ce02ad10c6f4686a476eb2a5de2988cd8b482f5e693db2880c84ad1c82f468ef03fe01b9d0feefe5d4ee741d1d16643d36b144e6261ed32311b3b6f312fac2f +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/0407cde92cfa42fa89ac83217ca0ec16 +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/032c831f1166a336551138939ac40eb2c68a048ce786c0c1403b879a20c1b706caac16d22560b2c7f2b3d6373986c347188675674116005ca251336ee048d09f +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/23418763b808371ee94772a90d501f4d +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/7867b843551457b11bda7821dd384c1c1cf23b80a308b2058a693de7b7da099f0b37eb0a6de2b84c04b625a68c60eea55138e200d5d6ec6f6af09bd7ce406a96 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/e3d33ae03c18affea74699bdc1fabb68 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/42013f4921de5a69ad857195ce5c19ad1bca3c920d79699e5501f1f4534ab132fabd422362b2b5056f5d182215d6c069db5df460bafa700903faf962cc00f77b +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/d40c1e8c0393213c6057c53a12f44175 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/fe7baa4de7490065ab7b953cc12f41462a24bcb49d0a4a64b23249e98e7569b19bb1cb455af2f76090e34066a7d3cdd7a48cae6515ce6c7a5c8486b0cacc5106 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/48541b90f715c4c86ee4da0570275947 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/7f2683fb98e80f12629f4ed3bea9fd59d32b7e7a9ed1699e782d8e238ff0915ecc61bf00adaf4597cfe41caf82cdca0f9be250f595f5f0bea6d8f77dba99eaf4 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/4547059eb905995667be48bf85d49911 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/7400fdabc924434ab4a4949248c3603887ac06ffd2f205ae33e14495d86cd4f816bbd1999eeafa0257f518df1e7f7c522f596e847a71dbfbfccff4859f50acc7 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/46267543cad6584d7b7b9fcc8f18f21d +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/0353d7d724be48d4185d3c181692970b7996f53f6a01723072aa5c94b53a8c5055faeed30df51659c252a46f4b941dec0cb24569323e3c85c166f14c5b7c8e9e +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/14dba2897a6e9d370fa9091c045375fc +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/10b79f9c059839f5b57fa8d2a381a034c4067262c4088bd354d14ea56bec097878069383aa9cfadaa09d73bd20fc348fb61662d863a8d62cb25d7af6b8e29858 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/eed836d1addeb10d0901f836724aff1e +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/e33eca424d1529a1fb23ba9cf7fac345ed1cfc8073c975b6b31ca44d2e8c3f5083af65433df009b22483dceb2e43149f3c1e8433681fec5fb812e1d5b4243ce4 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/d5ae9f9519341fdaabf62267c89461d2 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/6421aa5d1bd6f08ad43f59ed4dc1bef8b9b598ebbbd3e48149730f3bec3471f8e2c02ffb338427326924290b8f52ef9e626e3313448bc931a61d866c5dc544ae +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/fc1df521395362a5aaa2e2aeef707207 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/f2e5a08e3cae171242ae6a20d2d4838c1529ce042745dc466148b7bbc06896d94476fd05c7787e6e8641bea752dfc0e6b09e95b160bede600d20d2ad68e7705f +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/2338f8aa2696935f7460454e708ce308 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/5a4b0e97928c26eee16bbec4c3e69e55fa9c768101257c3e2f161118809c778aa0feaf21307198822c3172a58ed12ca0a49285b2941ed0b8f2b367e64ca1c51a +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/b393d2bf0d181d218130ac572c17d369 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/76e0f7caa24bb734c6f7542be9f834d5b912f082cb3c4c3c52a63e37d4b8c33dd94e576c43f4bee6c04bfb44af2f2b67ba70773fa52ad0de6c8c0059b3e51b83 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/23db836e6e4142f621862971017fe61e +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/c0b04f7fe5aabfe6af509c77a1f68e0bcfd14714758042fe502b968c4cc272156fc84c8b4c1ee574754bb2fddaa810f6a4215cbd164ddc11b697b3adaef09a81 diff --git a/doc/make.jl b/doc/make.jl index 75e3598ced6f7..04b8af595e58f 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -142,6 +142,7 @@ DevDocs = [ "devdocs/subarrays.md", "devdocs/isbitsunionarrays.md", "devdocs/sysimg.md", + "devdocs/pkgimg.md", "devdocs/llvm.md", "devdocs/stdio.md", "devdocs/boundscheck.md", diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md new file mode 100644 index 0000000000000..8230c4b91b338 --- /dev/null +++ b/doc/src/devdocs/pkgimg.md @@ -0,0 +1,48 @@ +# Package Images + +Julia package images provide object (native code) caches for Julia packages. +They are similar to Julia's [system image](@ref dev-sysimg) and support many of the same features. +In fact the underlying serialization format is the same, and the system image is the base image that the package images are build against. + +## High-level overview + +Package images are shared libraries that contain both code and data. Like `.ji` cache files, they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM-based compiler. + +The command line option `--pkgimages=no` can be used to turn off object caching for this session. Note that this means that cache files have to likely be regenerated. +See [`JULIA_MAX_NUM_PRECOMPILE_FILES`](@ref env-max-num-precompile-files) for the upper limit of variants Julia caches per default. + +!!! note + While the package images present themselves as native shared libraries, they are only an approximation thereof. You will not be able to link against them from a native program and they must be loaded from Julia. + + +## Linking + +Since the package images contain native code, we must run a linker over them before we can use them. You can set the environment variable `JULIA_VERBOSE_LINKING` to `true` to make the package image linking process verbose. + +Furthermore, we cannot assume that the user has a working system linker installed. Therefore, Julia ships with LLD, the LLVM linker, to provide a working out of the box experience. In `base/linking.jl`, we implement a limited interface to be able to link package images on all supported platforms. + +### Quirks +Despite LLD being a multi-platform linker, it does not provide a consistent interface across platforms. Furthermore, it is meant to be used from `clang` or +another compiler driver, we therefore reimplement some of the logic from `llvm-project/clang/lib/Driver/ToolChains`. Thankfully one can use `lld -flavor` to set lld to the right platform + +#### Windows +To avoid having to deal with `link.exe` we use `-flavor gnu`, effectively turning `lld` into a cross-linker from a mingw32 environment. Windows DLLs are required to contain a `_DllMainCRTStartup` function and to minimize our dependence on mingw32 libraries, we inject a stub definition ourselves. + +#### MacOS +Dynamic libraries on macOS need to link against `-lSystem`. On recent macOS versions, `-lSystem` is only available for linking when Xcode is available. +To that effect we link with `-undefined dynamic_lookup`. + +## Package images optimized for multiple microarchitectures +Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogenous environment, with a unified cache, +you can set the environment variable `JULIA_CPU_TARGET=generic` to multi-version the object caches. + +## Flags that impact package image creation and selection + +These are the Julia command line flags that impact cache selection. Package images +that were created with different flags will be rejected. + +- `-g`, `--debug-info`: Exact match required since it changes code generation. +- `--check-bounds`: Exact match required since it changes code generation. +- `--pkgimages`: To allow running without object caching enabled. +- `-O`, `--optimize`: Reject package images generated for a lower optimization level, + but allow for higher optimization levels to be loaded. diff --git a/doc/src/devdocs/sysimg.md b/doc/src/devdocs/sysimg.md index 5c976875846d3..a21e3ba265f9b 100644 --- a/doc/src/devdocs/sysimg.md +++ b/doc/src/devdocs/sysimg.md @@ -19,7 +19,7 @@ This operation is useful for multiple reasons. A user may: The [`PackageCompiler.jl` package](https://github.com/JuliaLang/PackageCompiler.jl) contains convenient wrapper functions to automate this process. -## System image optimized for multiple microarchitectures +## [System image optimized for multiple microarchitectures](@id sysimg-multi-versioning) The system image can be compiled simultaneously for multiple CPU microarchitectures under the same instruction set architecture (ISA). Multiple versions of the same function diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index f6e74a02b4b30..6ff9128264161 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -389,17 +389,18 @@ When a package with extensions is added to an environment, the `weakdeps` and `e are stored in the manifest file in the section for that package. The dependency lookup rules for a package are the same as for its "parent" except that the listed extension dependencies are also considered as dependencies. -### Package/Environment Preferences + +### [Package/Environment Preferences](@id preferences) Preferences are dictionaries of metadata that influence package behavior within an environment. -The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that a particular `.ji` file was built with the same preferences as the current environment before loading it. +The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that the precompilation files selected by Julia were built with the same preferences as the current environment before loading them. The public API for modifying Preferences is contained within the [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) package. Preferences are stored as TOML dictionaries within a `(Julia)LocalPreferences.toml` file next to the currently-active project. If a preference is "exported", it is instead stored within the `(Julia)Project.toml` instead. The intention is to allow shared projects to contain shared preferences, while allowing for users themselves to override those preferences with their own settings in the LocalPreferences.toml file, which should be .gitignored as the name implies. -Preferences that are accessed during compilation are automatically marked as compile-time preferences, and any change recorded to these preferences will cause the Julia compiler to recompile any cached precompilation `.ji` files for that module. -This is done by serializing the hash of all compile-time preferences during compilation, then checking that hash against the current environment when searching for the proper `.ji` file to load. +Preferences that are accessed during compilation are automatically marked as compile-time preferences, and any change recorded to these preferences will cause the Julia compiler to recompile any cached precompilation file(s) (`.ji` and corresponding `.so`, `.dll`, or `.dylib` files) for that module. +This is done by serializing the hash of all compile-time preferences during compilation, then checking that hash against the current environment when searching for the proper file(s) to load. Preferences can be set with depot-wide defaults; if package Foo is installed within your global environment and it has preferences set, these preferences will apply as long as your global environment is part of your `LOAD_PATH`. Preferences in environments higher up in the environment stack get overridden by the more proximal entries in the load path, ending with the currently active project. diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 4af3c05d51eb6..e35cbf5e313e7 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -88,7 +88,7 @@ There are various ways to run Julia code and provide options, similar to those a julia [switches] -- [programfile] [args...] ``` -The following is a complete list of command-line switches available when launching julia (a '*' marks the default value, if applicable): +The following is a complete list of command-line switches available when launching julia (a '*' marks the default value, if applicable; settings marked '($)' may trigger package precompilation): |Switch |Description| |:--- |:---| @@ -102,6 +102,7 @@ The following is a complete list of command-line switches available when launchi |`--handle-signals={yes*\|no}` |Enable or disable Julia's default signal handlers| |`--sysimage-native-code={yes*\|no}` |Use native code from system image if available| |`--compiled-modules={yes*\|no}` |Enable or disable incremental precompilation of modules| +|`--pkgimages={yes*\|no}` |Enable or disable usage of native code caching in the form of pkgimages| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| |`-L`, `--load ` |Load `` immediately on all processors| @@ -117,11 +118,11 @@ The following is a complete list of command-line switches available when launchi |`--warn-overwrite={yes\|no*}` |Enable or disable method overwrite warnings| |`--warn-scope={yes*\|no}` |Enable or disable warning for ambiguous top-level scope| |`-C`, `--cpu-target ` |Limit usage of CPU features up to ``; set to `help` to see the available options| -|`-O`, `--optimize={0,1,2*,3}` |Set the optimization level (level is 3 if `-O` is used without a level)| +|`-O`, `--optimize={0,1,2*,3}` |Set the optimization level (level is 3 if `-O` is used without a level) ($)| |`--min-optlevel={0*,1,2,3}` |Set the lower bound on per-module optimization| -|`-g {0,1*,2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level)| +|`-g`, `--debug-info={0,1*,2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level) ($)| |`--inline={yes\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| -|`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations| +|`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations ($)| |`--math-mode={ieee,fast}` |Disallow or enable unsafe floating point optimizations (overrides `@fastmath` declaration)| |`--code-coverage[={none*\|user\|all}]` |Count executions of source lines (omitting setting is equivalent to `user`)| |`--code-coverage=tracefile.info` |Append coverage information to the LCOV tracefile (filename supports format tokens).| diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index bc4a742365d69..f29e5b7aaf8f7 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -162,10 +162,14 @@ The absolute path `REPL.find_hist_file()` of the REPL's history file. If $(DEPOT_PATH[1])/logs/repl_history.jl ``` -### `JULIA_MAX_NUM_PRECOMPILE_FILES` +### [`JULIA_MAX_NUM_PRECOMPILE_FILES`](@id env-max-num-precompile-files) Sets the maximum number of different instances of a single package that are to be stored in the precompile cache (default = 10). +### `JULIA_VERBOSE_LINKING` + +If set to true, linker commands will be displayed during precompilation. + ## Pkg.jl ### `JULIA_CI` diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 6cbcc4fad6a65..a504f8e3511b2 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -265,8 +265,40 @@ julia> methods(+) ``` Multiple dispatch together with the flexible parametric type system give Julia its ability to -abstractly express high-level algorithms decoupled from implementation details, yet generate efficient, -specialized code to handle each case at run time. +abstractly express high-level algorithms decoupled from implementation details. + +## [Method specializations](@id man-method-specializations) + +When you create multiple methods of the same function, this is sometimes called +"specialization." In this case, you're specializing the *function* by adding additional +methods to it: each new method is a new specialization of the function. +As shown above, these specializations are returned by `methods`. + +There's another kind of specialization that occurs without programmer intervention: +Julia's compiler can automatically specialize the *method* for the specific argument types used. +Such specializations are *not* listed by `methods`, as this doesn't create new `Method`s, but tools like [`@code_typed`](@ref) allow you to inspect such specializations. + +For example, if you create a method + +``` +mysum(x::Real, y::Real) = x + y +``` + +you've given the function `mysum` one new method (possibly its only method), and that method takes any pair of `Real` number inputs. But if you then execute + +```julia-repl +julia> mysum(1, 2) +3 + +julia> mysum(1.0, 2.0) +3.0 +``` + +Julia will compile `mysum` twice, once for `x::Int, y::Int` and again for `x::Float64, y::Float64`. +The point of compiling twice is performance: the methods that get called for `+` (which `mysum` uses) vary depending on the specific types of `x` and `y`, and by compiling different specializations Julia can do all the method lookup ahead of time. This allows the program to run much more quickly, since it does not have to bother with method lookup while it is running. +Julia's automatic specialization allows you to write generic algorithms and expect that the compiler will generate efficient, specialized code to handle each case you need. + +In cases where the number of potential specializations might be effectively unlimited, Julia may avoid this default specialization. See [Be aware of when Julia avoids specializing](@ref) for more information. ## [Method Ambiguities](@id man-ambiguities) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index f0a9a5110ded4..90680828d2bc2 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -9,7 +9,7 @@ Modules in Julia help organize code into coherent units. They are delimited synt 2. Modules have facilities for detailed namespace management: each defines a set of names it `export`s, and can import names from other modules with `using` and `import` (we explain these below). -3. Modules can be precompiled for faster loading, and contain code for runtime initialization. +3. Modules can be precompiled for faster loading, and may contain code for runtime initialization. Typically, in larger Julia packages you will see module code organized into files, eg @@ -429,11 +429,14 @@ Large modules can take several seconds to load because executing all of the stat often involves compiling a large amount of code. Julia creates precompiled caches of the module to reduce this time. -The incremental precompiled module file are created and used automatically when using `import` -or `using` to load a module. This will cause it to be automatically compiled the first time -it is imported. Alternatively, you can manually call [`Base.compilecache(Base.identify_package("modulename"))`](@ref). The resulting -cache files will be stored in `DEPOT_PATH[1]/compiled/`. Subsequently, the module is automatically -recompiled upon `using` or `import` whenever any of its dependencies change; dependencies are modules it +Precompiled module files (sometimes called "cache files") are created and used automatically when `import` or `using` loads a module. If the cache file(s) do not yet exist, the module will be compiled and saved for future reuse. You can also manually call [`Base.compilecache(Base.identify_package("modulename"))`](@ref) to create these files without loading the module. The resulting +cache files will be stored in the `compiled` subfolder of `DEPOT_PATH[1]`. If nothing about your system changes, +such cache files will be used when you load the module with `import` or `using`. + +Precompilation cache files store definitions of modules, types, methods, and constants. They may also store method specializations and the code generated for them, but this typically requires that the developer add explicit [`precompile`](@ref) directives or execute workloads that force compilation during the package build. + +However, if you update the module's dependencies or change its source code, the module is automatically +recompiled upon `using` or `import`. Dependencies are modules it imports, the Julia build, files it includes, or explicit dependencies declared by [`include_dependency(path)`](@ref) in the module file(s). @@ -445,6 +448,7 @@ by the search logic in `require` matches the path that had created the precompil into account the set of dependencies already loaded into the current process and won't recompile those modules, even if their files change or disappear, in order to avoid creating incompatibilities between the running system and the precompile cache. +Finally, it takes account of changes in any [compile-time preferences](@ref preferences). If you know that a module is *not* safe to precompile (for example, for one of the reasons described below), you should @@ -589,6 +593,12 @@ A few other points to be aware of: It is sometimes helpful during module development to turn off incremental precompilation. The command line flag `--compiled-modules={yes|no}` enables you to toggle module precompilation on and off. When Julia is started with `--compiled-modules=no` the serialized modules in the compile cache -are ignored when loading modules and module dependencies. `Base.compilecache` can still be called +are ignored when loading modules and module dependencies. +More fine-grained control is available with `--pkgimages=no`, which suppresses only +native-code storage during precompilation. `Base.compilecache` can still be called manually. The state of this command line flag is passed to `Pkg.build` to disable automatic precompilation triggering when installing, updating, and explicitly building packages. + +You can also debug some precompilation failures with environment variables. Setting +`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of compiled +native code. See the **Developer Documentation** part of the Julia manual, where you will find further details in the section documenting Julia's internals under "Package Images". diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index 1a316e5fcf347..1eee23e163a77 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -525,7 +525,7 @@ at the time `k` is compiled. ### Be aware of when Julia avoids specializing -As a heuristic, Julia avoids automatically specializing on argument type parameters in three +As a heuristic, Julia avoids automatically [specializing](@ref man-method-specializations) on argument type parameters in three specific cases: `Type`, `Function`, and `Vararg`. Julia will always specialize when the argument is used within the method, but not if the argument is just passed through to another function. This usually has no performance impact at runtime and diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 7325adde8b060..6f9345ee18f82 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -61,7 +61,9 @@ using namespace llvm; #include "jitlayers.h" +#include "serialize.h" #include "julia_assert.h" +#include "codegen_shared.h" #define DEBUG_TYPE "julia_aotcompile" @@ -93,6 +95,7 @@ typedef struct { std::vector jl_sysimg_gvars; std::map> jl_fvar_map; std::vector jl_value_to_llvm; + std::vector jl_external_to_llvm; } jl_native_code_desc_t; extern "C" JL_DLLEXPORT @@ -118,6 +121,15 @@ void jl_get_llvm_gvs_impl(void *native_code, arraylist_t *gvs) memcpy(gvs->items, data->jl_value_to_llvm.data(), gvs->len * sizeof(void*)); } +extern "C" JL_DLLEXPORT +void jl_get_llvm_external_fns_impl(void *native_code, arraylist_t *external_fns) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + arraylist_grow(external_fns, data->jl_external_to_llvm.size()); + memcpy(external_fns->items, data->jl_external_to_llvm.data(), + external_fns->len * sizeof(jl_code_instance_t*)); +} + extern "C" JL_DLLEXPORT LLVMOrcThreadSafeModuleRef jl_get_llvm_module_impl(void *native_code) { @@ -248,13 +260,17 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance *ci_out = codeinst; } +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const); + // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup, and can // also be used be extern consumers like GPUCompiler.jl to obtain a module containing -// all reachable & inferrrable functions. The `policy` flag switches between the default -// mode `0`, the extern mode `1`. +// all reachable & inferrrable functions. +// The `policy` flag switches between the default mode `0` and the extern mode `1` used by GPUCompiler. +// `_imaging_mode` controls if raw pointers can be embedded (e.g. the code will be loaded into the same session). +// `_external_linkage` create linkages between pkgimages. extern "C" JL_DLLEXPORT -void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode) +void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode, int _external_linkage) { ++CreateNativeCalls; CreateNativeMax.updateMax(jl_array_len(methods)); @@ -289,6 +305,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm jl_codegen_params_t params(ctxt); params.params = cgparams; params.imaging = imaging; + params.external_linkage = _external_linkage; size_t compile_for[] = { jl_typeinf_world, jl_atomic_load_acquire(&jl_world_counter) }; for (int worlds = 0; worlds < 2; worlds++) { params.world = compile_for[worlds]; @@ -348,6 +365,39 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm } CreateNativeMethods += emitted.size(); + size_t offset = gvars.size(); + data->jl_external_to_llvm.resize(params.external_fns.size()); + + auto tbaa_const = tbaa_make_child_with_context(*ctxt.getContext(), "jtbaa_const", nullptr, true).first; + for (auto &extern_fn : params.external_fns) { + jl_code_instance_t *this_code = std::get<0>(extern_fn.first); + bool specsig = std::get<1>(extern_fn.first); + assert(specsig && "Error external_fns doesn't handle non-specsig yet"); + (void)specsig; + Function *F = extern_fn.second; + Module *M = F->getParent(); + + Type *T_funcp = F->getFunctionType()->getPointerTo(); + // Can't create a GC with type FunctionType. Alias also doesn't work + GlobalVariable *GV = new GlobalVariable(*M, T_funcp, false, + GlobalVariable::ExternalLinkage, + Constant::getNullValue(T_funcp), + F->getName()); + + + // Need to insert load instruction, thus we can't use replace all uses with + replaceUsesWithLoad(*F, [GV](Instruction &) { return GV; }, tbaa_const); + + assert(F->getNumUses() == 0); // declaration counts as use + GV->takeName(F); + F->eraseFromParent(); + + size_t idx = gvars.size() - offset; + assert(idx >= 0); + data->jl_external_to_llvm.at(idx) = this_code; + gvars.push_back(std::string(GV->getName())); + } + // clones the contents of the module `m` to the shadow_output collector // while examining and recording what kind of function pointer we have for (auto &def : emitted) { @@ -459,7 +509,7 @@ static void injectCRTAlias(Module &M, StringRef name, StringRef alias, FunctionT if (!target) { target = Function::Create(FT, Function::ExternalLinkage, alias, M); } - Function *interposer = Function::Create(FT, Function::WeakAnyLinkage, name, M); + Function *interposer = Function::Create(FT, Function::InternalLinkage, name, M); appendToCompilerUsed(M, {interposer}); llvm::IRBuilder<> builder(BasicBlock::Create(M.getContext(), "top", interposer)); @@ -477,7 +527,7 @@ extern "C" JL_DLLEXPORT void jl_dump_native_impl(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len) + const char *sysimg_data, size_t sysimg_len, ios_t *s) { JL_TIMING(NATIVE_DUMP); jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; @@ -589,7 +639,7 @@ void jl_dump_native_impl(void *native_code, } // do the actual work - auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name, StringRef asm_Name) { + auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name, StringRef asm_Name, bool inject_crt) { preopt.run(M, empty.MAM); if (bc_fname || obj_fname || asm_fname) { assert(!verifyModule(M, &errs())); @@ -597,22 +647,49 @@ void jl_dump_native_impl(void *native_code, assert(!verifyModule(M, &errs())); } - // We would like to emit an alias or an weakref alias to redirect these symbols - // but LLVM doesn't let us emit a GlobalAlias to a declaration... - // So for now we inject a definition of these functions that calls our runtime - // functions. We do so after optimization to avoid cloning these functions. - injectCRTAlias(M, "__gnu_h2f_ieee", "julia__gnu_h2f_ieee", - FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); - injectCRTAlias(M, "__extendhfsf2", "julia__gnu_h2f_ieee", - FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); - injectCRTAlias(M, "__gnu_f2h_ieee", "julia__gnu_f2h_ieee", - FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); - injectCRTAlias(M, "__truncsfhf2", "julia__gnu_f2h_ieee", - FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); - injectCRTAlias(M, "__truncdfhf2", "julia__truncdfhf2", - FunctionType::get(Type::getHalfTy(Context), { Type::getDoubleTy(Context) }, false)); + if (inject_crt) { + // We would like to emit an alias or an weakref alias to redirect these symbols + // but LLVM doesn't let us emit a GlobalAlias to a declaration... + // So for now we inject a definition of these functions that calls our runtime + // functions. We do so after optimization to avoid cloning these functions. + injectCRTAlias(M, "__gnu_h2f_ieee", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__extendhfsf2", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__gnu_f2h_ieee", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncsfhf2", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncdfhf2", "julia__truncdfhf2", + FunctionType::get(Type::getHalfTy(Context), { Type::getDoubleTy(Context) }, false)); + +#if defined(_OS_WINDOWS_) + // Windows expect that the function `_DllMainStartup` is present in an dll. + // Normal compilers use something like Zig's crtdll.c instead we provide a + // a stub implementation. + auto T_pvoid = Type::getInt8Ty(Context)->getPointerTo(); + auto T_int32 = Type::getInt32Ty(Context); + auto FT = FunctionType::get(T_int32, {T_pvoid, T_int32, T_pvoid}, false); + auto F = Function::Create(FT, Function::ExternalLinkage, "_DllMainCRTStartup", M); + F->setCallingConv(CallingConv::X86_StdCall); + + llvm::IRBuilder<> builder(BasicBlock::Create(M.getContext(), "top", F)); + builder.CreateRet(ConstantInt::get(T_int32, 1)); +#endif + } postopt.run(M, empty.MAM); + + // Get target by snooping on multiversioning + GlobalVariable *target_ids = M.getNamedGlobal("jl_dispatch_target_ids"); + if (s && target_ids) { + if(auto targets = dyn_cast(target_ids->getInitializer())) { + auto rawTargets = targets->getRawDataValues(); + write_int32(s, rawTargets.size()); + ios_write(s, rawTargets.data(), rawTargets.size()); + }; + } + emitter.run(M); if (unopt_bc_fname) @@ -625,7 +702,7 @@ void jl_dump_native_impl(void *native_code, emit_result(asm_Archive, asm_Buffer, asm_Name, outputs); }; - add_output(*dataM, "unopt.bc", "text.bc", "text.o", "text.s"); + add_output(*dataM, "unopt.bc", "text.bc", "text.o", "text.s", true); orc::ThreadSafeModule sysimage(std::make_unique("sysimage", Context), TSCtx); auto sysimageM = sysimage.getModuleUnlocked(); @@ -648,7 +725,7 @@ void jl_dump_native_impl(void *native_code, GlobalVariable::ExternalLinkage, len, "jl_system_image_size")); } - add_output(*sysimageM, "data.bc", "data.bc", "data.o", "data.s"); + add_output(*sysimageM, "data.bc", "data.bc", "data.o", "data.s", false); object::Archive::Kind Kind = getDefaultForHost(TheTriple); if (unopt_bc_fname) diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 01324e349f08f..e7b7d1fb791a5 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -12,8 +12,9 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len) UNAVAILABLE + const char *sysimg_data, size_t sysimg_len, ios_t *s) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE +JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE JL_DLLEXPORT void jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_method_asm_fallback(jl_method_instance_t *linfo, size_t world, @@ -31,10 +32,10 @@ JL_DLLEXPORT int jl_getFunctionInfo_fallback(jl_frame_t **frames, uintptr_t poin return 0; } -JL_DLLEXPORT void jl_register_fptrs_fallback(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs, +JL_DLLEXPORT void jl_register_fptrs_fallback(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n) { - (void)sysimage_base; (void)fptrs; (void)linfos; (void)n; + (void)image_base; (void)fptrs; (void)linfos; (void)n; } JL_DLLEXPORT jl_code_instance_t *jl_generate_fptr_fallback(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) @@ -66,7 +67,7 @@ JL_DLLEXPORT size_t jl_jit_total_bytes_fallback(void) return 0; } -JL_DLLEXPORT void *jl_create_native_fallback(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode) UNAVAILABLE +JL_DLLEXPORT void *jl_create_native_fallback(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode, int _external_linkage) UNAVAILABLE JL_DLLEXPORT void jl_dump_compiles_fallback(void *s) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 5b173d3930584..c01e340431e5f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1419,6 +1419,7 @@ class jl_codectx_t { jl_codegen_params_t &emission_context; llvm::MapVector call_targets; std::map &global_targets; + std::map, Function*> &external_calls; Function *f = NULL; // local var info. globals are not in here. std::vector slots; @@ -1455,6 +1456,7 @@ class jl_codectx_t { bool debug_enabled = false; bool use_cache = false; + bool external_linkage = false; const jl_cgparams_t *params = NULL; std::vector llvmcall_modules; @@ -1464,8 +1466,10 @@ class jl_codectx_t { emission_context(params), call_targets(), global_targets(params.globals), + external_calls(params.external_fns), world(params.world), use_cache(params.cache), + external_linkage(params.external_linkage), params(params.params) { } jl_typecache_t &types() { @@ -4028,9 +4032,17 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const std::string name; StringRef protoname; bool need_to_emit = true; - // TODO: We should check if the code is available externally - // and then emit a trampoline. - if (ctx.use_cache) { + bool cache_valid = ctx.use_cache; + bool external = false; + if (ctx.external_linkage) { + if (jl_object_in_image((jl_value_t*)codeinst)) { + // Target is present in another pkgimage + cache_valid = true; + external = true; + } + } + + if (cache_valid) { // optimization: emit the correct name immediately, if we know it // TODO: use `emitted` map here too to try to consolidate names? // WARNING: isspecsig is protected by the codegen-lock. If that lock is removed, then the isspecsig load needs to be properly atomically sequenced with this. @@ -4058,6 +4070,13 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, argv, nargs, &cc, &return_roots, rt); else result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, argv, nargs, rt); + if (external) { + assert(!need_to_emit); + auto calledF = jl_Module->getFunction(protoname); + assert(calledF); + // TODO: Check if already present? + ctx.external_calls[std::make_tuple(codeinst, specsig)] = calledF; + } handled = true; if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); @@ -5376,7 +5395,16 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod Function *theFunc; Value *theFarg; auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); - if (params.cache && invoke != NULL) { + + bool cache_valid = params.cache; + if (params.external_linkage) { + if (jl_object_in_image((jl_value_t*)codeinst)) { + // Target is present in another pkgimage + cache_valid = true; + } + } + + if (cache_valid && invoke != NULL) { StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, codeinst); theFunc = cast( M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(ctx.builder.getContext())).getCallee()); @@ -8273,12 +8301,12 @@ void jl_compile_workqueue( StringRef preal_decl = ""; bool preal_specsig = false; auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); - // TODO: available_extern - // We need to emit a trampoline that loads the target address in an extern_module from a GV - // Right now we will unecessarily emit a function we have already compiled in a native module - // again in a calling module. + bool cache_valid = params.cache; + if (params.external_linkage) { + cache_valid = jl_object_in_image((jl_value_t*)codeinst); + } // WARNING: isspecsig is protected by the codegen-lock. If that lock is removed, then the isspecsig load needs to be properly atomically sequenced with this. - if (params.cache && invoke != NULL) { + if (cache_valid && invoke != NULL) { auto fptr = jl_atomic_load_relaxed(&codeinst->specptr.fptr); if (invoke == jl_fptr_args_addr) { preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, codeinst); diff --git a/src/coverage.cpp b/src/coverage.cpp index 46363a7e9ac01..0dfb903798bfa 100644 --- a/src/coverage.cpp +++ b/src/coverage.cpp @@ -17,7 +17,7 @@ using namespace llvm; static int codegen_imaging_mode(void) { - return jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); + return jl_options.image_codegen || jl_generating_output(); } // Logging for code coverage and memory allocation diff --git a/src/debug-registry.h b/src/debug-registry.h index 3780bbee33718..165f0efa479e3 100644 --- a/src/debug-registry.h +++ b/src/debug-registry.h @@ -81,11 +81,11 @@ class JITDebugInfoRegistry ~Locked() JL_NOTSAFEPOINT = default; }; - struct sysimg_info_t { - uint64_t jl_sysimage_base; - jl_sysimg_fptrs_t sysimg_fptrs; - jl_method_instance_t **sysimg_fvars_linfo; - size_t sysimg_fvars_n; + struct image_info_t { + uint64_t base; + jl_image_fptrs_t fptrs; + jl_method_instance_t **fvars_linfo; + size_t fvars_n; }; struct libc_frames_t { @@ -122,7 +122,7 @@ class JITDebugInfoRegistry // that it came from (providing name, type signature, file info, etc.) Locked> codeinst_in_flight{}; - Locked sysimg_info{}; + Locked> image_info{}; Locked objfilemap{}; @@ -141,7 +141,7 @@ class JITDebugInfoRegistry std::function getLoadAddress, std::function lookupWriteAddress) JL_NOTSAFEPOINT; objectmap_t& getObjectMap() JL_NOTSAFEPOINT; - void set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT; - Locked::ConstLockT get_sysimg_info() const JL_NOTSAFEPOINT; + void add_image_info(image_info_t info) JL_NOTSAFEPOINT; + bool get_image_info(uint64_t base, image_info_t *info) const JL_NOTSAFEPOINT; Locked::LockT get_objfile_map() JL_NOTSAFEPOINT; }; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index fe5614100f9e3..997c04aff6445 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -109,13 +109,19 @@ JITDebugInfoRegistry::getObjectMap() JL_NOTSAFEPOINT return objectmap; } -void JITDebugInfoRegistry::set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT { - (**this->sysimg_info) = info; +void JITDebugInfoRegistry::add_image_info(image_info_t info) JL_NOTSAFEPOINT { + (**this->image_info)[info.base] = info; } -JITDebugInfoRegistry::Locked::ConstLockT -JITDebugInfoRegistry::get_sysimg_info() const JL_NOTSAFEPOINT { - return *this->sysimg_info; + +bool JITDebugInfoRegistry::get_image_info(uint64_t base, JITDebugInfoRegistry::image_info_t *info) const JL_NOTSAFEPOINT { + auto infos = *this->image_info; + auto it = infos->find(base); + if (it != infos->end()) { + *info = it->second; + return true; + } + return false; } JITDebugInfoRegistry::Locked::LockT @@ -680,10 +686,10 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info) std::move(SplitFile.get())); } extern "C" JL_DLLEXPORT -void jl_register_fptrs_impl(uint64_t sysimage_base, const jl_sysimg_fptrs_t *fptrs, +void jl_register_fptrs_impl(uint64_t image_base, const jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n) { - getJITDebugRegistry().set_sysimg_info({(uintptr_t) sysimage_base, *fptrs, linfos, n}); + getJITDebugRegistry().add_image_info({(uintptr_t) image_base, *fptrs, linfos, n}); } template @@ -694,12 +700,9 @@ static inline void ignoreError(T &err) JL_NOTSAFEPOINT #endif } -static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool insysimage, +static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool inimage, void **saddr, char **name, bool untrusted_dladdr) JL_NOTSAFEPOINT { - // Assume we only need base address for sysimg for now - if (!insysimage || !getJITDebugRegistry().get_sysimg_info()->sysimg_fptrs.base) - saddr = nullptr; bool needs_saddr = saddr && (!*saddr || untrusted_dladdr); bool needs_name = name && (!*name || untrusted_dladdr); // Try platform specific methods first since they are usually faster @@ -780,7 +783,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t } #ifdef _OS_WINDOWS_ // For ntdll and msvcrt since we are currently only parsing DWARF debug info through LLVM - if (!insysimage && needs_name) { + if (!inimage && needs_name) { static char frame_info_func[ sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)]; @@ -1012,7 +1015,7 @@ static object::SectionRef getModuleSectionForAddress(const object::ObjectFile *o bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t *slide, llvm::DIContext **context, - bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT + bool onlyImage, bool *isImage, uint64_t *_fbase, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT { *Section = object::SectionRef(); *context = NULL; @@ -1046,10 +1049,11 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * if (fname.empty()) // empirically, LoadedImageName might be missing fname = ModuleInfo.ImageName; DWORD64 fbase = ModuleInfo.BaseOfImage; - bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) + JITDebugInfoRegistry::image_info_t image_info; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image_info); + if (isImage) + *isImage = inimage; + if (onlyImage && !inimage) return false; // If we didn't find the filename before in the debug // info, use the dll name @@ -1057,6 +1061,8 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * jl_copy_str(filename, fname.data()); if (saddr) *saddr = NULL; + if (_fbase) + *_fbase = fbase; #else // ifdef _OS_WINDOWS_ Dl_info dlinfo; @@ -1095,16 +1101,19 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * fbase = (uintptr_t)dlinfo.dli_fbase; #endif StringRef fname; - bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); - if (saddr && !(insysimage && untrusted_dladdr)) + JITDebugInfoRegistry::image_info_t image_info; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image_info); + if (saddr && !(inimage && untrusted_dladdr)) *saddr = dlinfo.dli_saddr; - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) + if (isImage) + *isImage = inimage; + if (onlyImage && !inimage) return false; + if (_fbase) + *_fbase = fbase; // In case we fail with the debug info lookup, we at least still // have the function name, even if we don't have line numbers - if (name && !(insysimage && untrusted_dladdr)) + if (name && !(inimage && untrusted_dladdr)) jl_copy_str(name, dlinfo.dli_sname); if (filename) jl_copy_str(filename, dlinfo.dli_fname); @@ -1115,7 +1124,10 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * *context = entry.ctx; if (entry.obj) *Section = getModuleSectionForAddress(entry.obj, pointer + entry.slide); - get_function_name_and_base(*Section, pointer, entry.slide, insysimage, saddr, name, untrusted_dladdr); + // Assume we only need base address for sysimg for now + if (!inimage || !image_info.fptrs.base) + saddr = nullptr; + get_function_name_and_base(*Section, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); return true; } @@ -1144,34 +1156,36 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip object::SectionRef Section; llvm::DIContext *context = NULL; int64_t slide; - bool isSysImg; + bool isImage; void *saddr; - if (!jl_dylib_DI_for_fptr(pointer, &Section, &slide, &context, skipC, &isSysImg, &saddr, &frame0->func_name, &frame0->file_name)) { + uint64_t fbase; + if (!jl_dylib_DI_for_fptr(pointer, &Section, &slide, &context, skipC, &isImage, &fbase, &saddr, &frame0->func_name, &frame0->file_name)) { frame0->fromC = 1; return 1; } - frame0->fromC = !isSysImg; + frame0->fromC = !isImage; { - auto sysimg_locked = getJITDebugRegistry().get_sysimg_info(); - if (isSysImg && sysimg_locked->sysimg_fptrs.base && saddr) { - intptr_t diff = (uintptr_t)saddr - (uintptr_t)sysimg_locked->sysimg_fptrs.base; - for (size_t i = 0; i < sysimg_locked->sysimg_fptrs.nclones; i++) { - if (diff == sysimg_locked->sysimg_fptrs.clone_offsets[i]) { - uint32_t idx = sysimg_locked->sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask; - if (idx < sysimg_locked->sysimg_fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) - frame0->linfo = sysimg_locked->sysimg_fvars_linfo[idx]; + JITDebugInfoRegistry::image_info_t image; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image); + if (isImage && saddr && inimage) { + intptr_t diff = (uintptr_t)saddr - (uintptr_t)image.fptrs.base; + for (size_t i = 0; i < image.fptrs.nclones; i++) { + if (diff == image.fptrs.clone_offsets[i]) { + uint32_t idx = image.fptrs.clone_idxs[i] & jl_sysimg_val_mask; + if (idx < image.fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) + frame0->linfo = image.fvars_linfo[idx]; break; } } - for (size_t i = 0; i < sysimg_locked->sysimg_fvars_n; i++) { - if (diff == sysimg_locked->sysimg_fptrs.offsets[i]) { - frame0->linfo = sysimg_locked->sysimg_fvars_linfo[i]; + for (size_t i = 0; i < image.fvars_n; i++) { + if (diff == image.fptrs.offsets[i]) { + frame0->linfo = image.fvars_linfo[i]; break; } } } } - return lookup_pointer(Section, context, frames, pointer, slide, isSysImg, noInline); + return lookup_pointer(Section, context, frames, pointer, slide, isImage, noInline); } int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, diff --git a/src/debuginfo.h b/src/debuginfo.h index 5ea34350ac1fb..5b5cdcb82d534 100644 --- a/src/debuginfo.h +++ b/src/debuginfo.h @@ -6,7 +6,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, llvm::object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT; bool jl_dylib_DI_for_fptr(size_t pointer, llvm::object::SectionRef *Section, int64_t *slide, llvm::DIContext **context, - bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT; + bool onlyImage, bool *isImage, uint64_t* fbase, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT; static object::SectionedAddress makeAddress( llvm::object::SectionRef Section, uint64_t address) JL_NOTSAFEPOINT diff --git a/src/disasm.cpp b/src/disasm.cpp index cfc030f649fd6..5b510a24b33da 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -592,7 +592,7 @@ jl_value_t *jl_dump_fptr_asm_impl(uint64_t fptr, char raw_mc, const char* asm_va llvm::DIContext *context = NULL; if (!jl_DI_for_fptr(fptr, &symsize, &slide, &Section, &context)) { if (!jl_dylib_DI_for_fptr(fptr, &Section, &slide, &context, - false, NULL, NULL, NULL, NULL)) { + false, NULL, NULL, NULL, NULL, NULL)) { jl_printf(JL_STDERR, "WARNING: Unable to find function pointer\n"); return jl_pchar_to_string("", 0); } diff --git a/src/jitlayers.h b/src/jitlayers.h index 9e3fa6dc5711d..8edeec8929014 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -73,7 +73,7 @@ GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M); DataLayout jl_create_datalayout(TargetMachine &TM); static inline bool imaging_default() { - return jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); + return jl_options.image_codegen || jl_generating_output(); } struct OptimizationOptions { @@ -173,6 +173,7 @@ typedef struct _jl_codegen_params_t { // outputs std::vector> workqueue; std::map globals; + std::map, Function*> external_fns; std::map ditypes; std::map llvmtypes; DenseMap mergedConstants; @@ -200,6 +201,7 @@ typedef struct _jl_codegen_params_t { size_t world = 0; const jl_cgparams_t *params = &jl_default_cgparams; bool cache = false; + bool external_linkage = false; bool imaging; _jl_codegen_params_t(orc::ThreadSafeContext ctx) : tsctx(std::move(ctx)), tsctx_lock(tsctx.getLock()), imaging(imaging_default()) {} } jl_codegen_params_t; diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index a254fba5e2b28..dd38560af1414 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -128,6 +128,8 @@ XX(jl_voidpointer_type) \ XX(jl_void_type) \ XX(jl_weakref_type) \ + XX(jl_build_ids) \ + XX(jl_linkage_blobs) \ // Data symbols that are defined inside the public libjulia #define JL_EXPORTED_DATA_SYMBOLS(XX) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 2e5df94dc7200..8ea77bfe12be3 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -118,6 +118,7 @@ XX(jl_dlopen) \ XX(jl_dlsym) \ XX(jl_dump_host_cpu) \ + XX(jl_check_pkgimage_clones) \ XX(jl_egal) \ XX(jl_egal__bits) \ XX(jl_egal__special) \ @@ -394,6 +395,8 @@ XX(jl_queue_work) \ XX(jl_raise_debugger) \ XX(jl_readuntil) \ + XX(jl_cache_flags) \ + XX(jl_match_cache_flags) \ XX(jl_read_verify_header) \ XX(jl_realloc) \ XX(jl_register_newmeth_tracer) \ @@ -536,6 +539,7 @@ YY(jl_get_LLVM_VERSION) \ YY(jl_dump_native) \ YY(jl_get_llvm_gvs) \ + YY(jl_get_llvm_external_fns) \ YY(jl_dump_function_asm) \ YY(jl_LLVMCreateDisasm) \ YY(jl_LLVMDisasmInstruction) \ diff --git a/src/jloptions.c b/src/jloptions.c index da362e0054b3e..90bb39955ee42 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -71,6 +71,7 @@ JL_DLLEXPORT void jl_init_options(void) JL_OPTIONS_HANDLE_SIGNALS_ON, JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES, JL_OPTIONS_USE_COMPILED_MODULES_YES, + JL_OPTIONS_USE_PKGIMAGES_YES, NULL, // bind-to NULL, // output-bc NULL, // output-unopt-bc @@ -92,7 +93,7 @@ JL_DLLEXPORT void jl_init_options(void) static const char usage[] = "\n julia [switches] -- [programfile] [args...]\n\n"; static const char opts[] = - "Switches (a '*' marks the default value, if applicable):\n\n" + "Switches (a '*' marks the default value, if applicable; settings marked '($)' may trigger package precompilation):\n\n" " -v, --version Display version information\n" " -h, --help Print this message (--help-hidden for more)\n" " --help-hidden Uncommon options not shown by `-h`\n\n" @@ -107,7 +108,9 @@ static const char opts[] = " --sysimage-native-code={yes*|no}\n" " Use native code from system image if available\n" " --compiled-modules={yes*|no}\n" - " Enable or disable incremental precompilation of modules\n\n" + " Enable or disable incremental precompilation of modules\n" + " --pkgimages={yes*|no}\n" + " Enable or disable usage of native code caching in the form of pkgimages ($)\n\n" // actions " -e, --eval Evaluate \n" @@ -143,16 +146,16 @@ static const char opts[] = // code generation options " -C, --cpu-target Limit usage of CPU features up to ; set to `help` to see the available options\n" - " -O, --optimize={0,1,2*,3} Set the optimization level (level 3 if `-O` is used without a level)\n" + " -O, --optimize={0,1,2*,3} Set the optimization level (level 3 if `-O` is used without a level) ($)\n" " --min-optlevel={0*,1,2,3} Set a lower bound on the optimization level\n" #ifdef JL_DEBUG_BUILD - " -g, --debug-info=[{0,1,2*}] Set the level of debug info generation in the julia-debug build\n" + " -g, --debug-info=[{0,1,2*}] Set the level of debug info generation in the julia-debug build ($)\n" #else - " -g, --debug-info=[{0,1*,2}] Set the level of debug info generation (level 2 if `-g` is used without a level)\n" + " -g, --debug-info=[{0,1*,2}] Set the level of debug info generation (level 2 if `-g` is used without a level) ($)\n" #endif " --inline={yes*|no} Control whether inlining is permitted, including overriding @inline declarations\n" " --check-bounds={yes|no|auto*}\n" - " Emit bounds checks always, never, or respect @inbounds declarations\n" + " Emit bounds checks always, never, or respect @inbounds declarations ($)\n" #ifdef USE_POLLY " --polly={yes*|no} Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)\n" #endif @@ -239,6 +242,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_banner, opt_sysimage_native_code, opt_compiled_modules, + opt_pkgimages, opt_machine_file, opt_project, opt_bug_report, @@ -267,6 +271,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "sysimage", required_argument, 0, 'J' }, { "sysimage-native-code", required_argument, 0, opt_sysimage_native_code }, { "compiled-modules",required_argument, 0, opt_compiled_modules }, + { "pkgimages", required_argument, 0, opt_pkgimages }, { "cpu-target", required_argument, 0, 'C' }, { "procs", required_argument, 0, 'p' }, { "threads", required_argument, 0, 't' }, @@ -317,6 +322,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) const char **cmds = NULL; int codecov = JL_LOG_NONE; int malloclog = JL_LOG_NONE; + int pkgimage_explicit = 0; int argc = *argcp; char **argv = *argvp; char *endptr; @@ -444,6 +450,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --compiled-modules={yes|no} (%s)", optarg); break; + case opt_pkgimages: + pkgimage_explicit = 1; + if (!strcmp(optarg,"yes")) + jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_YES; + else if (!strcmp(optarg,"no")) + jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_NO; + else + jl_errorf("julia: invalid argument to --pkgimage={yes|no} (%s)", optarg); + break; case 'C': // cpu-target jl_options.cpu_target = strdup(optarg); if (!jl_options.cpu_target) @@ -803,6 +818,13 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) "This is a bug, please report it.", c); } } + if (codecov || malloclog) { + if (pkgimage_explicit && jl_options.use_pkgimages) { + jl_errorf("julia: Can't use --pkgimages=yes together " + "with --track-allocation or --code-coverage."); + } + jl_options.use_pkgimages = 0; + } jl_options.code_coverage = codecov; jl_options.malloc_log = malloclog; int proc_args = *argcp < optind ? *argcp : optind; diff --git a/src/jloptions.h b/src/jloptions.h index d7be95348f01f..d0aba777027e7 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -42,6 +42,7 @@ typedef struct { int8_t handle_signals; int8_t use_sysimage_native_code; int8_t use_compiled_modules; + int8_t use_pkgimages; const char *bindto; const char *outputbc; const char *outputunoptbc; diff --git a/src/julia.h b/src/julia.h index 16f55a019d958..84fecd5fe28c4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1762,7 +1762,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); JL_DLLEXPORT void jl_set_sysimg_so(void *handle); -JL_DLLEXPORT ios_t *jl_create_system_image(void *, jl_array_t *worklist); +JL_DLLEXPORT void jl_create_system_image(void **, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete); @@ -2182,6 +2182,9 @@ JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; #define JL_OPTIONS_USE_COMPILED_MODULES_YES 1 #define JL_OPTIONS_USE_COMPILED_MODULES_NO 0 +#define JL_OPTIONS_USE_PKGIMAGES_YES 1 +#define JL_OPTIONS_USE_PKGIMAGES_NO 0 + // Version information #include // Generated file diff --git a/src/julia_internal.h b/src/julia_internal.h index 5710d430dc8ac..a13e548b13b16 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -992,11 +992,12 @@ JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, char raw_mc, const char JL_DLLEXPORT jl_value_t *jl_dump_function_ir(jl_llvmf_dump_t *dump, char strip_ir_metadata, char dump_module, const char *debuginfo); JL_DLLEXPORT jl_value_t *jl_dump_function_asm(jl_llvmf_dump_t *dump, char raw_mc, const char* asm_variant, const char *debuginfo, char binary); -void *jl_create_native(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int policy, int imaging_mode); +void *jl_create_native(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int policy, int imaging_mode, int cache); void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len); + const char *sysimg_data, size_t sysimg_len, ios_t *s); void jl_get_llvm_gvs(void *native_code, arraylist_t *gvs); +void jl_get_llvm_external_fns(void *native_code, arraylist_t *gvs); JL_DLLEXPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx); @@ -1619,9 +1620,9 @@ extern JL_DLLEXPORT jl_sym_t *jl_sequentially_consistent_sym; JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order(jl_sym_t *order, char loading, char storing); JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order_checked(jl_sym_t *order, char loading, char storing); -struct _jl_sysimg_fptrs_t; +struct _jl_image_fptrs_t; -void jl_register_fptrs(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs, +void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n); void jl_write_coverage_data(const char*); void jl_write_malloc_log(void); diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 815ebfe7ed101..97867f1d9f471 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -47,6 +47,8 @@ extern Optional always_have_fma(Function&); extern Optional always_have_fp16(); +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const); + namespace { constexpr uint32_t clone_mask = JL_TARGET_CLONE_LOOP | JL_TARGET_CLONE_SIMD | JL_TARGET_CLONE_MATH | JL_TARGET_CLONE_CPU; @@ -266,8 +268,6 @@ struct CloneCtx { uint32_t get_func_id(Function *F); template Constant *rewrite_gv_init(const Stack& stack); - template - Value *rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before); std::pair get_reloc_slot(Function *F); Constant *get_ptrdiff32(Constant *ptr, Constant *base) const; template @@ -820,7 +820,7 @@ std::pair CloneCtx::get_reloc_slot(Function *F) } template -Value *CloneCtx::rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before) +static Value *rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before) { SmallVector args; uint32_t nlevel = stack.size(); @@ -879,40 +879,24 @@ void CloneCtx::fix_inst_uses() continue; auto orig_f = orig_funcs[i]; auto F = grp.base_func(orig_f); - bool changed; - do { - changed = false; - for (auto uses = ConstantUses(F, M); !uses.done(); uses.next()) { - auto info = uses.get_info(); - auto use_i = info.val; - auto use_f = use_i->getFunction(); - if (!use_f->getName().endswith(suffix)) + replaceUsesWithLoad(*F, [&](Instruction &I) -> GlobalVariable * { + uint32_t id; + GlobalVariable *slot; + auto use_f = I.getFunction(); + if (!use_f->getName().endswith(suffix)) + return nullptr; + std::tie(id, slot) = get_reloc_slot(orig_f); + + grp.relocs.insert(id); + for (auto &tgt: grp.clones) { + // The enclosing function of the use is cloned, + // no need to deal with this use on this target. + if (map_get(*tgt.vmap, use_f)) continue; - Instruction *insert_before = use_i; - if (auto phi = dyn_cast(use_i)) - insert_before = phi->getIncomingBlock(*info.use)->getTerminator(); - uint32_t id; - GlobalVariable *slot; - std::tie(id, slot) = get_reloc_slot(orig_f); - Instruction *ptr = new LoadInst(orig_f->getType(), slot, "", false, insert_before); - ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); - ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ptr->getContext(), None)); - use_i->setOperand(info.use->getOperandNo(), - rewrite_inst_use(uses.get_stack(), ptr, - insert_before)); - - grp.relocs.insert(id); - for (auto &tgt: grp.clones) { - // The enclosing function of the use is cloned, - // no need to deal with this use on this target. - if (map_get(*tgt.vmap, use_f)) - continue; - tgt.relocs.insert(id); - } - - changed = true; + tgt.relocs.insert(id); } - } while (changed); + return slot; + }, tbaa_const); } } } @@ -1202,6 +1186,30 @@ static RegisterPass X("JuliaMultiVersioning", "JuliaMulti } // anonymous namespace +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const) { + bool changed; + do { + changed = false; + for (auto uses = ConstantUses(&F, *F.getParent()); !uses.done(); uses.next()) { + auto info = uses.get_info(); + auto use_i = info.val; + GlobalVariable *slot = should_replace(*use_i); + if (!slot) + continue; + Instruction *insert_before = use_i; + if (auto phi = dyn_cast(use_i)) + insert_before = phi->getIncomingBlock(*info.use)->getTerminator(); + Instruction *ptr = new LoadInst(F.getType(), slot, "", false, insert_before); + ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); + ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ptr->getContext(), None)); + use_i->setOperand(info.use->getOperandNo(), + rewrite_inst_use(uses.get_stack(), ptr, + insert_before)); + changed = true; + } + } while (changed); +} + PreservedAnalyses MultiVersioning::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult(M).getManager(); diff --git a/src/precompile.c b/src/precompile.c index ebe7afae69f64..bfc123cf3fda8 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -10,6 +10,7 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "serialize.h" #ifdef __cplusplus extern "C" { @@ -20,8 +21,53 @@ JL_DLLEXPORT int jl_generating_output(void) return jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputji || jl_options.outputasm; } -static void *jl_precompile(int all); -static void *jl_precompile_worklist(jl_array_t *worklist); +void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { + // Write the source-text for the dependent files + if (udeps) { + // Go back and update the source-text position to point to the current position + int64_t posfile = ios_pos(f); + ios_seek(f, srctextpos); + write_uint64(f, posfile); + ios_seek_end(f); + // Each source-text file is written as + // int32: length of abspath + // char*: abspath + // uint64: length of src text + // char*: src text + // At the end we write int32(0) as a terminal sentinel. + size_t len = jl_array_len(udeps); + ios_t srctext; + for (size_t i = 0; i < len; i++) { + jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); + jl_value_t *depmod = jl_fieldref(deptuple, 0); // module + // Dependencies declared with `include_dependency` are excluded + // because these may not be Julia code (and could be huge) + if (depmod != (jl_value_t*)jl_main_module) { + jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath + const char *depstr = jl_string_data(dep); + if (!depstr[0]) + continue; + ios_t *srctp = ios_file(&srctext, depstr, 1, 0, 0, 0); + if (!srctp) { + jl_printf(JL_STDERR, "WARNING: could not cache source text for \"%s\".\n", + jl_string_data(dep)); + continue; + } + size_t slen = jl_string_len(dep); + write_int32(f, slen); + ios_write(f, depstr, slen); + posfile = ios_pos(f); + write_uint64(f, 0); // placeholder for length of this file in bytes + uint64_t filelen = (uint64_t) ios_copyall(f, &srctext); + ios_close(&srctext); + ios_seek(f, posfile); + write_uint64(f, filelen); + ios_seek_end(f); + } + } + } + write_int32(f, 0); // mark the end of the source text +} JL_DLLEXPORT void jl_write_compiler_output(void) { @@ -35,7 +81,8 @@ JL_DLLEXPORT void jl_write_compiler_output(void) } jl_array_t *worklist = jl_module_init_order; - JL_GC_PUSH1(&worklist); + jl_array_t *udeps = NULL; + JL_GC_PUSH2(&worklist, &udeps); jl_module_init_order = jl_alloc_vec_any(0); int i, l = jl_array_len(worklist); for (i = 0; i < l; i++) { @@ -59,49 +106,54 @@ JL_DLLEXPORT void jl_write_compiler_output(void) assert(jl_precompile_toplevel_module == NULL); void *native_code = NULL; - if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm) { - if (jl_options.incremental) - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - native_code = jl_options.incremental ? jl_precompile_worklist(worklist) : jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); - if (jl_options.incremental) - jl_precompile_toplevel_module = NULL; - } - if (jl_options.incremental) { - if (jl_options.outputbc || jl_options.outputunoptbc) - jl_printf(JL_STDERR, "WARNING: incremental output to a .bc file is not implemented\n"); - if (jl_options.outputasm) - jl_printf(JL_STDERR, "WARNING: incremental output to a .s file is not implemented\n"); - if (jl_options.outputo) { - jl_printf(JL_STDERR, "WARNING: incremental output to a .o file is not implemented\n"); - } - } + bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm; - ios_t *s = jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL); + bool_t emit_split = jl_options.outputji && emit_native; - if (jl_options.outputji) { - ios_t f; - if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) - jl_errorf("cannot open system image file \"%s\" for writing", jl_options.outputji); - ios_write(&f, (const char*)s->buf, (size_t)s->size); - ios_close(&f); - } + ios_t *s = NULL; + ios_t *z = NULL; + int64_t srctextpos = 0 ; + jl_create_system_image(&native_code, jl_options.incremental ? worklist : NULL, emit_split, + &s, &z, &udeps, &srctextpos); + if (!emit_split) + z = s; + + // jl_dump_native writes the clone_targets into `s` + // We need to postpone the srctext writing after that. if (native_code) { jl_dump_native(native_code, jl_options.outputbc, jl_options.outputunoptbc, jl_options.outputo, jl_options.outputasm, - (const char*)s->buf, (size_t)s->size); + (const char*)z->buf, (size_t)z->size, s); jl_postoutput_hook(); } + if ((jl_options.outputji || emit_native) && jl_options.incremental) { + write_srctext(s, udeps, srctextpos); + } + + if (jl_options.outputji) { + ios_t f; + if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) + jl_errorf("cannot open system image file \"%s\" for writing", jl_options.outputji); + ios_write(&f, (const char*)s->buf, (size_t)s->size); + ios_close(&f); + } + if (s) { ios_close(s); free(s); } + if (emit_split) { + ios_close(z); + free(z); + } + for (size_t i = 0; i < jl_current_modules.size; i += 2) { if (jl_current_modules.table[i + 1] != HT_NOTFOUND) { jl_printf(JL_STDERR, "\nWARNING: detected unclosed module: "); @@ -112,296 +164,6 @@ JL_DLLEXPORT void jl_write_compiler_output(void) JL_GC_POP(); } -// f{<:Union{...}}(...) is a common pattern -// and expanding the Union may give a leaf function -static void _compile_all_tvar_union(jl_value_t *methsig) -{ - int tvarslen = jl_subtype_env_size(methsig); - jl_value_t *sigbody = methsig; - jl_value_t **roots; - JL_GC_PUSHARGS(roots, 1 + 2 * tvarslen); - jl_value_t **env = roots + 1; - int *idx = (int*)alloca(sizeof(int) * tvarslen); - int i; - for (i = 0; i < tvarslen; i++) { - assert(jl_is_unionall(sigbody)); - idx[i] = 0; - env[2 * i] = (jl_value_t*)((jl_unionall_t*)sigbody)->var; - env[2 * i + 1] = jl_bottom_type; // initialize the list with Union{}, since T<:Union{} is always a valid option - sigbody = ((jl_unionall_t*)sigbody)->body; - } - - for (i = 0; i < tvarslen; /* incremented by inner loop */) { - jl_value_t **sig = &roots[0]; - JL_TRY { - // TODO: wrap in UnionAll for each tvar in env[2*i + 1] ? - // currently doesn't matter much, since jl_compile_hint doesn't work on abstract types - *sig = (jl_value_t*)jl_instantiate_type_with(sigbody, env, tvarslen); - } - JL_CATCH { - goto getnext; // sigh, we found an invalid type signature. should we warn the user? - } - if (!jl_has_concrete_subtype(*sig)) - goto getnext; // signature wouldn't be callable / is invalid -- skip it - if (jl_is_concrete_type(*sig)) { - if (jl_compile_hint((jl_tupletype_t *)*sig)) - goto getnext; // success - } - - getnext: - for (i = 0; i < tvarslen; i++) { - jl_tvar_t *tv = (jl_tvar_t*)env[2 * i]; - if (jl_is_uniontype(tv->ub)) { - size_t l = jl_count_union_components(tv->ub); - size_t j = idx[i]; - if (j == l) { - env[2 * i + 1] = jl_bottom_type; - idx[i] = 0; - } - else { - jl_value_t *ty = jl_nth_union_component(tv->ub, j); - if (!jl_is_concrete_type(ty)) - ty = (jl_value_t*)jl_new_typevar(tv->name, tv->lb, ty); - env[2 * i + 1] = ty; - idx[i] = j + 1; - break; - } - } - else { - env[2 * i + 1] = (jl_value_t*)tv; - } - } - } - JL_GC_POP(); -} - -// f(::Union{...}, ...) is a common pattern -// and expanding the Union may give a leaf function -static void _compile_all_union(jl_value_t *sig) -{ - jl_tupletype_t *sigbody = (jl_tupletype_t*)jl_unwrap_unionall(sig); - size_t count_unions = 0; - size_t i, l = jl_svec_len(sigbody->parameters); - jl_svec_t *p = NULL; - jl_value_t *methsig = NULL; - - for (i = 0; i < l; i++) { - jl_value_t *ty = jl_svecref(sigbody->parameters, i); - if (jl_is_uniontype(ty)) - ++count_unions; - else if (ty == jl_bottom_type) - return; // why does this method exist? - else if (jl_is_datatype(ty) && !jl_has_free_typevars(ty) && - ((!jl_is_kind(ty) && ((jl_datatype_t*)ty)->isconcretetype) || - ((jl_datatype_t*)ty)->name == jl_type_typename)) - return; // no amount of union splitting will make this a leaftype signature - } - - if (count_unions == 0 || count_unions >= 6) { - _compile_all_tvar_union(sig); - return; - } - - int *idx = (int*)alloca(sizeof(int) * count_unions); - for (i = 0; i < count_unions; i++) { - idx[i] = 0; - } - - JL_GC_PUSH2(&p, &methsig); - int idx_ctr = 0, incr = 0; - while (!incr) { - p = jl_alloc_svec_uninit(l); - for (i = 0, idx_ctr = 0, incr = 1; i < l; i++) { - jl_value_t *ty = jl_svecref(sigbody->parameters, i); - if (jl_is_uniontype(ty)) { - assert(idx_ctr < count_unions); - size_t l = jl_count_union_components(ty); - size_t j = idx[idx_ctr]; - jl_svecset(p, i, jl_nth_union_component(ty, j)); - ++j; - if (incr) { - if (j == l) { - idx[idx_ctr] = 0; - } - else { - idx[idx_ctr] = j; - incr = 0; - } - } - ++idx_ctr; - } - else { - jl_svecset(p, i, ty); - } - } - methsig = (jl_value_t*)jl_apply_tuple_type(p); - methsig = jl_rewrap_unionall(methsig, sig); - _compile_all_tvar_union(methsig); - } - - JL_GC_POP(); -} - -static int compile_all_collect__(jl_typemap_entry_t *ml, void *env) -{ - jl_array_t *allmeths = (jl_array_t*)env; - jl_method_t *m = ml->func.method; - if (m->source) { - // method has a non-generated definition; can be compiled generically - jl_array_ptr_1d_push(allmeths, (jl_value_t*)m); - } - return 1; -} - -static int compile_all_collect_(jl_methtable_t *mt, void *env) -{ - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), compile_all_collect__, env); - return 1; -} - -static void jl_compile_all_defs(jl_array_t *mis) -{ - jl_array_t *allmeths = jl_alloc_vec_any(0); - JL_GC_PUSH1(&allmeths); - - jl_foreach_reachable_mtable(compile_all_collect_, allmeths); - - size_t i, l = jl_array_len(allmeths); - for (i = 0; i < l; i++) { - jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); - if (jl_is_datatype(m->sig) && jl_isa_compileable_sig((jl_tupletype_t*)m->sig, jl_emptysvec, m)) { - // method has a single compilable specialization, e.g. its definition - // signature is concrete. in this case we can just hint it. - jl_compile_hint((jl_tupletype_t*)m->sig); - } - else { - // first try to create leaf signatures from the signature declaration and compile those - _compile_all_union(m->sig); - - // finally, compile a fully generic fallback that can work for all arguments - jl_method_instance_t *unspec = jl_get_unspecialized(m); - if (unspec) - jl_array_ptr_1d_push(mis, (jl_value_t*)unspec); - } - } - - JL_GC_POP(); -} - -static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closure) -{ - assert(jl_is_method_instance(mi)); - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - while (codeinst) { - int do_compile = 0; - if (jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return) { - jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); - if (inferred && - inferred != jl_nothing && - jl_ir_flag_inferred((jl_array_t*)inferred) && - (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { - do_compile = 1; - } - else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { - do_compile = 1; - } - } - if (do_compile) { - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - return 1; - } - codeinst = jl_atomic_load_relaxed(&codeinst->next); - } - return 1; -} - -static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) -{ - jl_method_t *m = def->func.method; - if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { - // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled - jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - } - else { - jl_svec_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); - size_t i, l = jl_svec_len(specializations); - for (i = 0; i < l; i++) { - jl_value_t *mi = jl_svecref(specializations, i); - if (mi != jl_nothing) - precompile_enq_specialization_((jl_method_instance_t*)mi, closure); - } - } - if (m->ccallable) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); - return 1; -} - -static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) -{ - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); -} - -static void *jl_precompile_(jl_array_t *m) -{ - jl_array_t *m2 = NULL; - jl_method_instance_t *mi = NULL; - JL_GC_PUSH2(&m2, &mi); - m2 = jl_alloc_vec_any(0); - for (size_t i = 0; i < jl_array_len(m); i++) { - jl_value_t *item = jl_array_ptr_ref(m, i); - if (jl_is_method_instance(item)) { - mi = (jl_method_instance_t*)item; - size_t min_world = 0; - size_t max_world = ~(size_t)0; - if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) - mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); - if (mi) - jl_array_ptr_1d_push(m2, (jl_value_t*)mi); - } - else { - assert(jl_is_simplevector(item)); - assert(jl_svec_len(item) == 2); - jl_array_ptr_1d_push(m2, item); - } - } - void *native_code = jl_create_native(m2, NULL, NULL, 0, 1); - JL_GC_POP(); - return native_code; -} - -static void *jl_precompile(int all) -{ - // array of MethodInstances and ccallable aliases to include in the output - jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); - if (all) - jl_compile_all_defs(m); - jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); - void *native_code = jl_precompile_(m); - JL_GC_POP(); - return native_code; -} - -static void *jl_precompile_worklist(jl_array_t *worklist) -{ - if (!worklist) - return NULL; - // this "found" array will contain function - // type signatures that were inferred but haven't been compiled - jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); - size_t i, nw = jl_array_len(worklist); - for (i = 0; i < nw; i++) { - jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); - assert(jl_is_module(mod)); - foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); - } - void *native_code = jl_precompile_(m); - JL_GC_POP(); - return native_code; -} - #ifdef __cplusplus } #endif diff --git a/src/precompile_utils.c b/src/precompile_utils.c new file mode 100644 index 0000000000000..f251d00f76cfd --- /dev/null +++ b/src/precompile_utils.c @@ -0,0 +1,306 @@ +// f{<:Union{...}}(...) is a common pattern +// and expanding the Union may give a leaf function +static void _compile_all_tvar_union(jl_value_t *methsig) +{ + int tvarslen = jl_subtype_env_size(methsig); + jl_value_t *sigbody = methsig; + jl_value_t **roots; + JL_GC_PUSHARGS(roots, 1 + 2 * tvarslen); + jl_value_t **env = roots + 1; + int *idx = (int*)alloca(sizeof(int) * tvarslen); + int i; + for (i = 0; i < tvarslen; i++) { + assert(jl_is_unionall(sigbody)); + idx[i] = 0; + env[2 * i] = (jl_value_t*)((jl_unionall_t*)sigbody)->var; + env[2 * i + 1] = jl_bottom_type; // initialize the list with Union{}, since T<:Union{} is always a valid option + sigbody = ((jl_unionall_t*)sigbody)->body; + } + + for (i = 0; i < tvarslen; /* incremented by inner loop */) { + jl_value_t **sig = &roots[0]; + JL_TRY { + // TODO: wrap in UnionAll for each tvar in env[2*i + 1] ? + // currently doesn't matter much, since jl_compile_hint doesn't work on abstract types + *sig = (jl_value_t*)jl_instantiate_type_with(sigbody, env, tvarslen); + } + JL_CATCH { + goto getnext; // sigh, we found an invalid type signature. should we warn the user? + } + if (!jl_has_concrete_subtype(*sig)) + goto getnext; // signature wouldn't be callable / is invalid -- skip it + if (jl_is_concrete_type(*sig)) { + if (jl_compile_hint((jl_tupletype_t *)*sig)) + goto getnext; // success + } + + getnext: + for (i = 0; i < tvarslen; i++) { + jl_tvar_t *tv = (jl_tvar_t*)env[2 * i]; + if (jl_is_uniontype(tv->ub)) { + size_t l = jl_count_union_components(tv->ub); + size_t j = idx[i]; + if (j == l) { + env[2 * i + 1] = jl_bottom_type; + idx[i] = 0; + } + else { + jl_value_t *ty = jl_nth_union_component(tv->ub, j); + if (!jl_is_concrete_type(ty)) + ty = (jl_value_t*)jl_new_typevar(tv->name, tv->lb, ty); + env[2 * i + 1] = ty; + idx[i] = j + 1; + break; + } + } + else { + env[2 * i + 1] = (jl_value_t*)tv; + } + } + } + JL_GC_POP(); +} + +// f(::Union{...}, ...) is a common pattern +// and expanding the Union may give a leaf function +static void _compile_all_union(jl_value_t *sig) +{ + jl_tupletype_t *sigbody = (jl_tupletype_t*)jl_unwrap_unionall(sig); + size_t count_unions = 0; + size_t i, l = jl_svec_len(sigbody->parameters); + jl_svec_t *p = NULL; + jl_value_t *methsig = NULL; + + for (i = 0; i < l; i++) { + jl_value_t *ty = jl_svecref(sigbody->parameters, i); + if (jl_is_uniontype(ty)) + ++count_unions; + else if (ty == jl_bottom_type) + return; // why does this method exist? + else if (jl_is_datatype(ty) && !jl_has_free_typevars(ty) && + ((!jl_is_kind(ty) && ((jl_datatype_t*)ty)->isconcretetype) || + ((jl_datatype_t*)ty)->name == jl_type_typename)) + return; // no amount of union splitting will make this a leaftype signature + } + + if (count_unions == 0 || count_unions >= 6) { + _compile_all_tvar_union(sig); + return; + } + + int *idx = (int*)alloca(sizeof(int) * count_unions); + for (i = 0; i < count_unions; i++) { + idx[i] = 0; + } + + JL_GC_PUSH2(&p, &methsig); + int idx_ctr = 0, incr = 0; + while (!incr) { + p = jl_alloc_svec_uninit(l); + for (i = 0, idx_ctr = 0, incr = 1; i < l; i++) { + jl_value_t *ty = jl_svecref(sigbody->parameters, i); + if (jl_is_uniontype(ty)) { + assert(idx_ctr < count_unions); + size_t l = jl_count_union_components(ty); + size_t j = idx[idx_ctr]; + jl_svecset(p, i, jl_nth_union_component(ty, j)); + ++j; + if (incr) { + if (j == l) { + idx[idx_ctr] = 0; + } + else { + idx[idx_ctr] = j; + incr = 0; + } + } + ++idx_ctr; + } + else { + jl_svecset(p, i, ty); + } + } + methsig = (jl_value_t*)jl_apply_tuple_type(p); + methsig = jl_rewrap_unionall(methsig, sig); + _compile_all_tvar_union(methsig); + } + + JL_GC_POP(); +} + +static int compile_all_collect__(jl_typemap_entry_t *ml, void *env) +{ + jl_array_t *allmeths = (jl_array_t*)env; + jl_method_t *m = ml->func.method; + if (m->source) { + // method has a non-generated definition; can be compiled generically + jl_array_ptr_1d_push(allmeths, (jl_value_t*)m); + } + return 1; +} + +static int compile_all_collect_(jl_methtable_t *mt, void *env) +{ + jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), compile_all_collect__, env); + return 1; +} + +static void jl_compile_all_defs(jl_array_t *mis) +{ + jl_array_t *allmeths = jl_alloc_vec_any(0); + JL_GC_PUSH1(&allmeths); + + jl_foreach_reachable_mtable(compile_all_collect_, allmeths); + + size_t i, l = jl_array_len(allmeths); + for (i = 0; i < l; i++) { + jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); + if (jl_is_datatype(m->sig) && jl_isa_compileable_sig((jl_tupletype_t*)m->sig, jl_emptysvec, m)) { + // method has a single compilable specialization, e.g. its definition + // signature is concrete. in this case we can just hint it. + jl_compile_hint((jl_tupletype_t*)m->sig); + } + else { + // first try to create leaf signatures from the signature declaration and compile those + _compile_all_union(m->sig); + + // finally, compile a fully generic fallback that can work for all arguments + jl_method_instance_t *unspec = jl_get_unspecialized(m); + if (unspec) + jl_array_ptr_1d_push(mis, (jl_value_t*)unspec); + } + } + + JL_GC_POP(); +} + +static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closure) +{ + assert(jl_is_method_instance(mi)); + jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); + while (codeinst) { + int do_compile = 0; + if (jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return) { + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); + if (inferred && + inferred != jl_nothing && + jl_ir_flag_inferred((jl_array_t*)inferred) && + (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { + do_compile = 1; + } + else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { + do_compile = 1; + } + } + if (do_compile) { + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + return 1; + } + codeinst = jl_atomic_load_relaxed(&codeinst->next); + } + return 1; +} + +static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) +{ + jl_method_t *m = def->func.method; + if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { + // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled + jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + } + else { + jl_svec_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); + size_t i, l = jl_svec_len(specializations); + for (i = 0; i < l; i++) { + jl_value_t *mi = jl_svecref(specializations, i); + if (mi != jl_nothing) + precompile_enq_specialization_((jl_method_instance_t*)mi, closure); + } + } + if (m->ccallable) + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); + return 1; +} + +static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) +{ + return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); +} + +static void *jl_precompile_(jl_array_t *m, int external_linkage) +{ + jl_array_t *m2 = NULL; + jl_method_instance_t *mi = NULL; + JL_GC_PUSH2(&m2, &mi); + m2 = jl_alloc_vec_any(0); + for (size_t i = 0; i < jl_array_len(m); i++) { + jl_value_t *item = jl_array_ptr_ref(m, i); + if (jl_is_method_instance(item)) { + mi = (jl_method_instance_t*)item; + size_t min_world = 0; + size_t max_world = ~(size_t)0; + if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); + if (mi) + jl_array_ptr_1d_push(m2, (jl_value_t*)mi); + } + else { + assert(jl_is_simplevector(item)); + assert(jl_svec_len(item) == 2); + jl_array_ptr_1d_push(m2, item); + } + } + void *native_code = jl_create_native(m2, NULL, NULL, 0, 1, external_linkage); + JL_GC_POP(); + return native_code; +} + +static void *jl_precompile(int all) +{ + // array of MethodInstances and ccallable aliases to include in the output + jl_array_t *m = jl_alloc_vec_any(0); + JL_GC_PUSH1(&m); + if (all) + jl_compile_all_defs(m); + jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); + void *native_code = jl_precompile_(m, 0); + JL_GC_POP(); + return native_code; +} + +static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_specializations) +{ + if (!worklist) + return NULL; + // this "found" array will contain function + // type signatures that were inferred but haven't been compiled + jl_array_t *m = jl_alloc_vec_any(0); + JL_GC_PUSH1(&m); + size_t i, n = jl_array_len(worklist); + for (i = 0; i < n; i++) { + jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); + assert(jl_is_module(mod)); + foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); + } + n = jl_array_len(extext_methods); + for (i = 0; i < n; i++) { + jl_method_t *method = (jl_method_t*)jl_array_ptr_ref(extext_methods, i); + assert(jl_is_method(method)); + jl_svec_t *specializations = jl_atomic_load_relaxed(&method->specializations); + size_t j, l = jl_svec_len(specializations); + for (j = 0; j < l; j++) { + jl_value_t *mi = jl_svecref(specializations, j); + if (mi != jl_nothing) + precompile_enq_specialization_((jl_method_instance_t*)mi, m); + } + } + n = jl_array_len(new_specializations); + for (i = 0; i < n; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i); + precompile_enq_specialization_(ci->def, m); + } + void *native_code = jl_precompile_(m, 1); + JL_GC_POP(); + return native_code; +} diff --git a/src/processor.cpp b/src/processor.cpp index df114b4d80257..13b40ec4f7363 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -621,9 +621,9 @@ static inline std::vector> &get_cmdline_targets(F &&feature_cb) // Load sysimg, use the `callback` for dispatch and perform all relocations // for the selected target. template -static inline jl_sysimg_fptrs_t parse_sysimg(void *hdl, F &&callback) +static inline jl_image_fptrs_t parse_sysimg(void *hdl, F &&callback) { - jl_sysimg_fptrs_t res = {nullptr, 0, nullptr, 0, nullptr, nullptr}; + jl_image_fptrs_t res = {nullptr, 0, nullptr, 0, nullptr, nullptr}; // .data base char *data_base; diff --git a/src/processor.h b/src/processor.h index ac00f8874141b..d1b18cb72b084 100644 --- a/src/processor.h +++ b/src/processor.h @@ -135,7 +135,7 @@ JL_DLLEXPORT int jl_test_cpu_feature(jl_cpu_feature_t feature); static const uint32_t jl_sysimg_tag_mask = 0x80000000u; static const uint32_t jl_sysimg_val_mask = ~((uint32_t)0x80000000u); -typedef struct _jl_sysimg_fptrs_t { +typedef struct _jl_image_fptrs_t { // base function pointer const char *base; // number of functions @@ -153,7 +153,7 @@ typedef struct _jl_sysimg_fptrs_t { const int32_t *clone_offsets; // sorted indices of the cloned functions (including the tag bit) const uint32_t *clone_idxs; -} jl_sysimg_fptrs_t; +} jl_image_fptrs_t; /** * Initialize the processor dispatch system with sysimg `hdl` (also initialize the sysimg itself). @@ -165,14 +165,15 @@ typedef struct _jl_sysimg_fptrs_t { * * Return the data about the function pointers selected. */ -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl); -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl); +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl); +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl); // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); // Dump the name and feature set of the host CPU // For debugging only JL_DLLEXPORT void jl_dump_host_cpu(void); +JL_DLLEXPORT void jl_check_pkgimage_clones(char* data); JL_DLLEXPORT int32_t jl_set_zero_subnormals(int8_t isZero); JL_DLLEXPORT int32_t jl_get_zero_subnormals(void); diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index f7a112993e3e5..3e7b22caf00d4 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -1802,14 +1802,14 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) return jl_cstr_to_string(host_cpu_name().c_str()); } -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); @@ -1818,6 +1818,11 @@ jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) return parse_sysimg(hdl, pkgimg_init_cb); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) { ensure_jit_target(imaging); diff --git a/src/processor_fallback.cpp b/src/processor_fallback.cpp index 3160bd0ba6750..c1353e1bb43b0 100644 --- a/src/processor_fallback.cpp +++ b/src/processor_fallback.cpp @@ -112,14 +112,14 @@ get_llvm_target_str(const TargetData<1> &data) using namespace Fallback; -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); @@ -170,6 +170,11 @@ JL_DLLEXPORT void jl_dump_host_cpu(void) jl_safe_printf("Features: %s\n", jl_get_cpu_features_llvm().c_str()); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + extern "C" int jl_test_cpu_feature(jl_cpu_feature_t) { return 0; diff --git a/src/processor_x86.cpp b/src/processor_x86.cpp index b73838a55777e..6f064ddd47d19 100644 --- a/src/processor_x86.cpp +++ b/src/processor_x86.cpp @@ -1019,19 +1019,24 @@ JL_DLLEXPORT void jl_dump_host_cpu(void) cpus, ncpu_names); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) { return jl_cstr_to_string(host_cpu_name().c_str()); } -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); diff --git a/src/staticdata.c b/src/staticdata.c index 786d0c966693b..eea59a8cffceb 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -90,6 +90,7 @@ External links: #include "julia_assert.h" #include "staticdata_utils.c" +#include "precompile_utils.c" #ifdef __cplusplus extern "C" { @@ -322,7 +323,7 @@ typedef struct { uint64_t base; uintptr_t *gvars_base; int32_t *gvars_offsets; - jl_sysimg_fptrs_t fptrs; + jl_image_fptrs_t fptrs; } jl_image_t; // array of definitions for the predefined function pointers @@ -365,6 +366,7 @@ typedef struct { jl_array_t *link_ids_relocs; jl_array_t *link_ids_gctags; jl_array_t *link_ids_gvars; + jl_array_t *link_ids_external_fnvars; jl_ptls_t ptls; htable_t callers_with_edges; jl_image_t *image; @@ -851,7 +853,6 @@ static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT ios_write(s, zeros, nb); } - static void write_pointer(ios_t *s) JL_NOTSAFEPOINT { assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value"); @@ -1077,6 +1078,24 @@ static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAF } } +static void record_external_fns(jl_serializer_state *s, arraylist_t *external_fns) JL_NOTSAFEPOINT +{ + if (!s->incremental) { + assert(external_fns->len == 0); + (void) external_fns; + return; + } + + // We could call jl_queue_for_serialization here, but that should + // always be a no-op. +#ifndef JL_NDEBUG + for (size_t i = 0; i < external_fns->len; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)external_fns->items[i]; + assert(jl_object_in_image((jl_value_t*)ci)); + } +#endif +} + jl_value_t *jl_find_ptr = NULL; // The main function for serializing all the items queued in `serialization_order` // (They are also stored in `serialization_queue` which is order-preserving, unlike the hash table used @@ -1567,7 +1586,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) } // Compute target location at deserialization -static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) +static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) JL_NOTSAFEPOINT { enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET); size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1)); @@ -1829,20 +1848,20 @@ static jl_value_t *jl_delayed_reloc(jl_serializer_state *s, uintptr_t offset) JL static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) { - jl_sysimg_fptrs_t fvars = image->fptrs; + jl_image_fptrs_t fvars = image->fptrs; // make these NULL now so we skip trying to restore GlobalVariable pointers later image->gvars_base = NULL; image->fptrs.base = NULL; if (fvars.base == NULL) return; - int sysimg_fvars_max = s->fptr_record->size / sizeof(void*); + int img_fvars_max = s->fptr_record->size / sizeof(void*); size_t i; uintptr_t base = (uintptr_t)&s->s->buf[0]; // These will become MethodInstance references, but they start out as a list of // offsets into `s` for CodeInstances jl_method_instance_t **linfos = (jl_method_instance_t**)&s->fptr_record->buf[0]; uint32_t clone_idx = 0; - for (i = 0; i < sysimg_fvars_max; i++) { + for (i = 0; i < img_fvars_max; i++) { reloc_t offset = *(reloc_t*)&linfos[i]; linfos[i] = NULL; if (offset != 0) { @@ -1877,12 +1896,13 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) } } // Tell LLVM about the native code - jl_register_fptrs(image->base, &fvars, linfos, sysimg_fvars_max); + jl_register_fptrs(image->base, &fvars, linfos, img_fvars_max); } -static void write_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT +static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_NOTSAFEPOINT { - ios_ensureroom(s->gvar_record, globals->len * sizeof(reloc_t)); + size_t len = globals->len + external_fns->len; + ios_ensureroom(s->gvar_record, len * sizeof(reloc_t)); for (size_t i = 0; i < globals->len; i++) { void *g = globals->items[i]; if (jl_is_binding((uintptr_t)g)) { @@ -1903,10 +1923,17 @@ static void write_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFE write_reloc_t(s->gvar_record, reloc); record_uniquing(s, (jl_value_t*)g, ((i << 2) | 2)); // mark as gvar && !tag } + for (size_t i = 0; i < external_fns->len; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)external_fns->items[i]; + uintptr_t item = backref_id(s, (void*)ci, s->link_ids_external_fnvars); + uintptr_t reloc = get_reloc_for_item(item, 0); + write_reloc_t(s->gvar_record, reloc); + } + return globals->len + 1; } // Pointer relocation for native-code referenced global variables -static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image) +static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint32_t external_fns_begin) { if (image->gvars_base == NULL) return; @@ -1915,17 +1942,24 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image) uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; - int link_index = 0; + int gvar_link_index = 0; + int external_fns_link_index = 0; for (i = 0; i < l; i++) { uintptr_t offset = gvars[i]; - uintptr_t v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &link_index); + uintptr_t v = 0; + if (i < external_fns_begin) { + v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &gvar_link_index); + } else { + v = get_item_for_reloc(s, base, size, offset, s->link_ids_external_fnvars, &external_fns_link_index); + } uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); *gv = v; } - assert(!s->link_ids_gvars || link_index == jl_array_len(s->link_ids_gvars)); + assert(!s->link_ids_gvars || gvar_link_index == jl_array_len(s->link_ids_gvars)); + assert(!s->link_ids_external_fnvars || external_fns_link_index == jl_array_len(s->link_ids_external_fnvars)); } -static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image) +static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image, uint32_t external_fns_begin) { if (image->gvars_base == NULL) return; @@ -1934,8 +1968,14 @@ static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image) for (i = 0; i < l; i++) { uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); uintptr_t v = *gv; - if (!jl_is_binding(v)) - v = (uintptr_t)jl_as_global_root((jl_value_t*)v); + if (i < external_fns_begin) { + if (!jl_is_binding(v)) + v = (uintptr_t)jl_as_global_root((jl_value_t*)v); + } else { + jl_code_instance_t *codeinst = (jl_code_instance_t*) v; + assert(codeinst && codeinst->isspecsig); + v = (uintptr_t)codeinst->specptr.fptr; + } *gv = v; } } @@ -2287,13 +2327,18 @@ static void jl_save_system_image_to_stream(ios_t *f, s.link_ids_relocs = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, 0); + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, 0); htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; arraylist_t gvars; + arraylist_t external_fns; arraylist_new(&gvars, 0); - if (native_functions) + arraylist_new(&external_fns, 0); + if (native_functions) { jl_get_llvm_gvs(native_functions, &gvars); + jl_get_llvm_external_fns(native_functions, &external_fns); + } if (worklist == NULL) { // empty!(Core.ARGS) @@ -2359,6 +2404,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_serialize_reachable(&s); // step 1.2: now that we have marked all bindings (badly), ensure all gvars are part of the sysimage record_gvars(&s, &gvars); + record_external_fns(&s, &external_fns); jl_serialize_reachable(&s); // step 1.3: prune (garbage collect) some special weak references from // built-in type caches @@ -2372,10 +2418,11 @@ static void jl_save_system_image_to_stream(ios_t *f, } } + uint32_t external_fns_begin = 0; { // step 2: build all the sysimg sections write_padding(&sysimg, sizeof(uintptr_t)); jl_write_values(&s); - write_gvars(&s, &gvars); + external_fns_begin = write_gvars(&s, &gvars, &external_fns); jl_write_relocations(&s); } @@ -2475,6 +2522,9 @@ static void jl_save_system_image_to_stream(ios_t *f, ios_write(f, (char*)jl_array_data(s.link_ids_relocs), jl_array_len(s.link_ids_relocs)*sizeof(uint64_t)); write_uint32(f, jl_array_len(s.link_ids_gvars)); ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars)*sizeof(uint64_t)); + write_uint32(f, jl_array_len(s.link_ids_external_fnvars)); + ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars)*sizeof(uint64_t)); + write_uint32(f, external_fns_begin); jl_write_arraylist(s.s, &s.ccallable_list); } // Write the build_id key @@ -2492,6 +2542,7 @@ static void jl_save_system_image_to_stream(ios_t *f, arraylist_free(&s.relocs_list); arraylist_free(&s.gctags_list); arraylist_free(&gvars); + arraylist_free(&external_fns); htable_free(&field_replace); if (worklist) htable_free(&external_objects); @@ -2512,9 +2563,8 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a assert(jl_precompile_toplevel_module == NULL); jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - write_header(f); - // last word of the header is the checksumpos - *checksumpos = ios_pos(f) - sizeof(uint64_t); + *checksumpos = write_header(f, 0); + write_uint8(f, jl_cache_flags()); // write description of contents (name, uuid, buildid) write_worklist_for_header(f, worklist); // Determine unique (module, abspath, mtime) dependencies for the files defining modules in the worklist @@ -2528,88 +2578,96 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a write_mod_list(f, *mod_array); } - -JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data, jl_array_t *worklist) +JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *worklist, bool_t emit_split, + ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos) { jl_gc_collect(JL_GC_FULL); jl_gc_collect(JL_GC_INCREMENTAL); // sweep finalizers JL_TIMING(SYSIMG_DUMP); + // iff emit_split + // write header and src_text to one file f/s + // write systemimg to a second file ff/z jl_task_t *ct = jl_current_task; ios_t *f = (ios_t*)malloc_s(sizeof(ios_t)); ios_mem(f, 0); - jl_array_t *mod_array = NULL, *udeps = NULL, *extext_methods = NULL, *new_specializations = NULL; + + ios_t *ff = NULL; + if (emit_split) { + ff = (ios_t*)malloc_s(sizeof(ios_t)); + ios_mem(ff, 0); + } else { + ff = f; + } + + jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_specializations = NULL; jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; - JL_GC_PUSH7(&mod_array, &udeps, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); - int64_t srctextpos = 0; int64_t checksumpos = 0; + int64_t checksumpos_ff = 0; int64_t datastartpos = 0; + JL_GC_PUSH6(&mod_array, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); if (worklist) { - jl_write_header_for_incremental(f, worklist, &mod_array, &udeps, &srctextpos, &checksumpos); + jl_write_header_for_incremental(f, worklist, &mod_array, udeps, srctextpos, &checksumpos); + if (emit_split) { + checksumpos_ff = write_header(ff, 1); + write_uint8(ff, jl_cache_flags()); + write_mod_list(ff, mod_array); + } else { + checksumpos_ff = checksumpos; + } jl_gc_enable_finalizers(ct, 0); // make sure we don't run any Julia code concurrently after this point jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); - write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); - datastartpos = ios_pos(f); + + // Generate _native_data` + if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm) { + jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); + *_native_data = jl_precompile_worklist(worklist, extext_methods, new_specializations); + jl_precompile_toplevel_module = NULL; + } + + if (!emit_split) { + write_int32(f, 0); // No clone_targets + write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); + } else { + write_padding(ff, LLT_ALIGN(ios_pos(ff), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(ff)); + } + datastartpos = ios_pos(ff); + } else { + *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); } - native_functions = _native_data; - jl_save_system_image_to_stream(f, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); + native_functions = *_native_data; + jl_save_system_image_to_stream(ff, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); native_functions = NULL; if (worklist) { jl_gc_enable_finalizers(ct, 1); // make sure we don't run any Julia code concurrently before this point + jl_precompile_toplevel_module = NULL; + } + + if (worklist) { // Go back and update the checksum in the header - int64_t dataendpos = ios_pos(f); - uint32_t checksum = jl_crc32c(0, &f->buf[datastartpos], dataendpos - datastartpos); - ios_seek(f, checksumpos); - write_uint64(f, checksum | ((uint64_t)0xfafbfcfd << 32)); - ios_seek(f, srctextpos); - write_uint64(f, dataendpos); - // Write the source-text for the dependent files - // Go back and update the source-text position to point to the current position - if (udeps) { - ios_seek_end(f); - // Each source-text file is written as - // int32: length of abspath - // char*: abspath - // uint64: length of src text - // char*: src text - // At the end we write int32(0) as a terminal sentinel. - size_t len = jl_array_len(udeps); - ios_t srctext; - for (size_t i = 0; i < len; i++) { - jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *depmod = jl_fieldref(deptuple, 0); // module - // Dependencies declared with `include_dependency` are excluded - // because these may not be Julia code (and could be huge) - if (depmod != (jl_value_t*)jl_main_module) { - jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath - const char *depstr = jl_string_data(dep); - if (!depstr[0]) - continue; - ios_t *srctp = ios_file(&srctext, depstr, 1, 0, 0, 0); - if (!srctp) { - jl_printf(JL_STDERR, "WARNING: could not cache source text for \"%s\".\n", - jl_string_data(dep)); - continue; - } - size_t slen = jl_string_len(dep); - write_int32(f, slen); - ios_write(f, depstr, slen); - int64_t posfile = ios_pos(f); - write_uint64(f, 0); // placeholder for length of this file in bytes - uint64_t filelen = (uint64_t) ios_copyall(f, &srctext); - ios_close(&srctext); - ios_seek(f, posfile); - write_uint64(f, filelen); - ios_seek_end(f); - } - } + int64_t dataendpos = ios_pos(ff); + uint32_t checksum = jl_crc32c(0, &ff->buf[datastartpos], dataendpos - datastartpos); + ios_seek(ff, checksumpos_ff); + write_uint64(ff, checksum | ((uint64_t)0xfafbfcfd << 32)); + write_uint64(ff, datastartpos); + write_uint64(ff, dataendpos); + ios_seek(ff, dataendpos); + + // Write the checksum to the split header if necessary + if (emit_split) { + int64_t cur = ios_pos(f); + ios_seek(f, checksumpos); + write_uint64(f, checksum | ((uint64_t)0xfafbfcfd << 32)); + ios_seek(f, cur); + // Next we will write the clone_targets and afterwards the srctext } - write_int32(f, 0); // mark the end of the source text - jl_precompile_toplevel_module = NULL; } JL_GC_POP(); - return f; + *s = f; + if (emit_split) + *z = ff; + return; } JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src); @@ -2672,7 +2730,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.ptls = jl_current_task->ptls; arraylist_new(&s.relocs_list, 0); arraylist_new(&s.gctags_list, 0); - s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = NULL; + s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = s.link_ids_external_fnvars = NULL; jl_value_t **const*const tags = get_tags(); htable_t new_dt_objs; htable_new(&new_dt_objs, 0); @@ -2764,6 +2822,12 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_gvars); ios_read(f, (char*)jl_array_data(s.link_ids_gvars), nlinks_gvars * sizeof(uint64_t)); } + size_t nlinks_external_fnvars = read_uint32(f); + if (nlinks_external_fnvars > 0) { + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_external_fnvars); + ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint64_t)); + } + uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); if (s.incremental) { assert(restored && init_order && extext_methods && new_specializations && method_roots_list && ext_targets && edges); @@ -2777,6 +2841,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } s.s = NULL; + // step 3: apply relocations assert(!ios_eof(f)); jl_read_symbols(&s); @@ -2793,7 +2858,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl (void)sizeof_tags; jl_read_reloclist(&s, s.link_ids_relocs, 0); // general relocs // s.link_ids_gvars will be processed in `jl_update_all_gvars` - jl_update_all_gvars(&s, image); // gvars relocs + // s.link_ids_external_fns will be processed in `jl_update_all_gvars` + jl_update_all_gvars(&s, image, external_fns_begin); // gvars relocs if (s.incremental) { jl_read_arraylist(s.relocs, &s.uniquing_types); jl_read_arraylist(s.relocs, &s.uniquing_objs); @@ -3102,7 +3168,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl arraylist_free(&s.fixup_objs); if (s.incremental) - jl_root_new_gvars(&s, image); + jl_root_new_gvars(&s, image, external_fns_begin); ios_close(&relocs); ios_close(&const_data); ios_close(&gvar_record); @@ -3172,21 +3238,26 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl jl_gc_enable(en); } -static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos) +static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos, int64_t *datastartpos) { - if (ios_eof(f) || 0 == (*checksum = jl_read_verify_header(f)) || (*checksum >> 32 != 0xfafbfcfd)) { + uint8_t pkgimage = 0; + if (ios_eof(f) || 0 == (*checksum = jl_read_verify_header(f, &pkgimage, dataendpos, datastartpos)) || (*checksum >> 32 != 0xfafbfcfd)) { return jl_get_exceptionf(jl_errorexception_type, "Precompile file header verification checks failed."); } - { // skip past the mod list + uint8_t flags = read_uint8(f); + if (pkgimage && !jl_match_cache_flags(flags)) { + return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch"); + } + if (!pkgimage) { + // skip past the worklist size_t len; while ((len = read_int32(f))) ios_skip(f, len + 3 * sizeof(uint64_t)); - } - { // skip past the dependency list + // skip past the dependency list size_t deplen = read_uint64(f); ios_skip(f, deplen - sizeof(uint64_t)); - *dataendpos = read_uint64(f); + read_uint64(f); // where is this write coming from? } // verify that the system state is valid @@ -3198,10 +3269,14 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im { uint64_t checksum = 0; int64_t dataendpos = 0; - jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos); + int64_t datastartpos = 0; + jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos, &datastartpos); + if (verify_fail) return verify_fail; + assert(datastartpos > 0 && datastartpos < dataendpos); + jl_value_t *restored = NULL; jl_array_t *init_order = NULL, *extext_methods = NULL, *new_specializations = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; jl_svec_t *cachesizes_sv = NULL; @@ -3212,11 +3287,9 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im { // make a permanent in-memory copy of f (excluding the header) ios_bufmode(f, bm_none); JL_SIGATOMIC_BEGIN(); - size_t len_begin = LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT); - assert(len_begin > 0 && len_begin < dataendpos); - size_t len = dataendpos - len_begin; + size_t len = dataendpos - datastartpos; char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - ios_seek(f, len_begin); + ios_seek(f, datastartpos); if (ios_readall(f, sysimg, len) != len || jl_crc32c(0, sysimg, len) != (uint32_t)checksum) { restored = jl_get_exceptionf(jl_errorexception_type, "Error reading system image file."); JL_SIGATOMIC_END(); @@ -3333,7 +3406,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_SIGATOMIC_END(); } -JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods) +JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int complete) { void *pkgimg_handle = jl_dlopen(fname, JL_RTLD_LAZY); if (!pkgimg_handle) { @@ -3357,9 +3430,9 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j if (!jl_dlsym(pkgimg_handle, "jl_sysimg_gvars_base", (void **)&pkgimage.gvars_base, 0)) { pkgimage.gvars_base = NULL; } + jl_dlsym(pkgimg_handle, "jl_sysimg_gvars_offsets", (void **)&pkgimage.gvars_offsets, 1); pkgimage.gvars_offsets += 1; - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, 0); void *pgcstack_func_slot; jl_dlsym(pkgimg_handle, "jl_pgcstack_func_slot", &pgcstack_func_slot, 0); @@ -3373,6 +3446,20 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j *tls_offset_idx = (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset); } + #ifdef _OS_WINDOWS_ + pkgimage.base = (intptr_t)pkgimg_handle; + #else + Dl_info dlinfo; + if (dladdr((void*)pkgimage.gvars_base, &dlinfo) != 0) { + pkgimage.base = (intptr_t)dlinfo.dli_fbase; + } + else { + pkgimage.base = 0; + } + #endif + + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, complete); + return mod; } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index dea511b1e0c98..220a354847dfe 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -611,11 +611,46 @@ static void write_mod_list(ios_t *s, jl_array_t *a) write_int32(s, 0); } +JL_DLLEXPORT uint8_t jl_cache_flags(void) +{ + // ??OOCDDP + uint8_t flags = 0; + flags |= (jl_options.use_pkgimages & 1); + flags |= (jl_options.debug_level & 3) << 1; + flags |= (jl_options.check_bounds & 1) << 2; + flags |= (jl_options.opt_level & 3) << 4; + // NOTES: + // In contrast to check-bounds, inline has no "observable effect" + return flags; +} + +JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) +{ + // 1. Check which flags are relevant + uint8_t current_flags = jl_cache_flags(); + uint8_t supports_pkgimage = (current_flags & 1); + uint8_t is_pkgimage = (flags & 1); + + // For .ji packages ignore other flags + if (!supports_pkgimage && !is_pkgimage) { + return 1; + } + + // 2. Check all flags that must be exact + uint8_t mask = (1 << 4)-1; + if ((flags & mask) != (current_flags & mask)) + return 0; + // 3. allow for higher optimization flags in cache + flags >>= 4; + current_flags >>= 4; + return flags >= current_flags; +} + // "magic" string and version header of .ji file static const int JI_FORMAT_VERSION = 12; static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature static const uint16_t BOM = 0xFEFF; // byte-order marker -static void write_header(ios_t *s) +static int64_t write_header(ios_t *s, uint8_t pkgimage) { ios_write(s, JI_MAGIC, strlen(JI_MAGIC)); write_uint16(s, JI_FORMAT_VERSION); @@ -627,7 +662,12 @@ static void write_header(ios_t *s) const char *branch = jl_git_branch(), *commit = jl_git_commit(); ios_write(s, branch, strlen(branch)+1); ios_write(s, commit, strlen(commit)+1); + write_uint8(s, pkgimage); + int64_t checksumpos = ios_pos(s); write_uint64(s, 0); // eventually will hold checksum for the content portion of this (build_id.hi) + write_uint64(s, 0); // eventually will hold dataendpos + write_uint64(s, 0); // eventually will hold datastartpos + return checksumpos; } // serialize information about the result of deserializing this file @@ -1206,9 +1246,10 @@ static int readstr_verify(ios_t *s, const char *str, int include_null) return 1; } -JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s) +JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s, uint8_t *pkgimage, int64_t *dataendpos, int64_t *datastartpos) { uint16_t bom; + uint64_t checksum = 0; if (readstr_verify(s, JI_MAGIC, 0) && read_uint16(s) == JI_FORMAT_VERSION && ios_read(s, (char *) &bom, 2) == 2 && bom == BOM && @@ -1218,6 +1259,11 @@ JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s) readstr_verify(s, JULIA_VERSION_STRING, 1) && readstr_verify(s, jl_git_branch(), 1) && readstr_verify(s, jl_git_commit(), 1)) - return read_uint64(s); - return 0; + { + *pkgimage = read_uint8(s); + checksum = read_uint64(s); + *datastartpos = (int64_t)read_uint64(s); + *dataendpos = (int64_t)read_uint64(s); + } + return checksum; } diff --git a/stdlib/CompilerSupportLibraries_jll/Project.toml b/stdlib/CompilerSupportLibraries_jll/Project.toml index b072831326627..fc5883cc79802 100644 --- a/stdlib/CompilerSupportLibraries_jll/Project.toml +++ b/stdlib/CompilerSupportLibraries_jll/Project.toml @@ -4,7 +4,7 @@ uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" # NOTE: When updating this, also make sure to update the value # `CSL_NEXT_GLIBCXX_VERSION` in `deps/csl.mk`, to properly disable # automatic usage of BB-built CSLs on extremely up-to-date systems! -version = "1.0.1+0" +version = "1.0.2+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/compiler/contextual.jl b/test/compiler/contextual.jl index 75cbcf2de37fc..79285f62b0947 100644 --- a/test/compiler/contextual.jl +++ b/test/compiler/contextual.jl @@ -209,7 +209,11 @@ try @test length(Bar.mt) == 1 finally rm(load_path, recursive=true, force=true) - rm(depot_path, recursive=true, force=true) + try + rm(depot_path, force=true, recursive=true) + catch err + @show err + end filter!((≠)(load_path), LOAD_PATH) filter!((≠)(depot_path), DEPOT_PATH) end diff --git a/test/loading.jl b/test/loading.jl index d52a7246abe7c..497dfaed4af18 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -746,7 +746,11 @@ for env in keys(envs) rm(env, force=true, recursive=true) end for depot in depots - rm(depot, force=true, recursive=true) + try + rm(depot, force=true, recursive=true) + catch err + @show err + end end append!(empty!(LOAD_PATH), saved_load_path) diff --git a/test/precompile.jl b/test/precompile.jl index 688286a113b55..8aa09efe3f417 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -28,8 +28,18 @@ function precompile_test_harness(@nospecialize(f), separate::Bool) pushfirst!(DEPOT_PATH, load_cache_path) f(load_path) finally - rm(load_path, recursive=true, force=true) - separate && rm(load_cache_path, recursive=true, force=true) + try + rm(load_path, force=true, recursive=true) + catch err + @show err + end + if separate + try + rm(load_cache_path, force=true, recursive=true) + catch err + @show err + end + end filter!((≠)(load_path), LOAD_PATH) separate && filter!((≠)(load_cache_path), DEPOT_PATH) end @@ -318,11 +328,16 @@ precompile_test_harness(false) do dir cachedir = joinpath(dir, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachedir2 = joinpath(dir2, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachefile = joinpath(cachedir, "$Foo_module.ji") + if Base.JLOptions().use_pkgimages == 1 + ocachefile = Base.ocachefile_from_cachefile(cachefile) + else + ocachefile = nothing + end # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_warn "@ccallable was already defined for this method name" begin @test_logs (:warn, "Replacing module `$Foo_module`") begin - m = Base._require_from_serialized(Base.PkgId(Foo), cachefile) + m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) @test isa(m, Module) end end @@ -343,7 +358,7 @@ precompile_test_harness(false) do dir @test string(Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" @test string(Base.Docs.doc(Foo.Bar)) == "Bar module\n" - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) discard_module = mod_fl_mt -> (mod_fl_mt.filename, mod_fl_mt.mtime) @test modules == [ Base.PkgId(Foo) => Base.module_build_id(Foo) % UInt64 ] @test map(x -> x.filename, deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] @@ -378,7 +393,7 @@ precompile_test_harness(false) do dir ), ) @test discard_module.(deps) == deps1 - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile; srcfiles_only=true) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile; srcfiles_only=true) @test map(x -> x.filename, deps) == [Foo_file] @test current_task()(0x01, 0x4000, 0x30031234) == 2 @@ -441,7 +456,7 @@ precompile_test_harness(false) do dir """) Nest = Base.require(Main, Nest_module) cachefile = joinpath(cachedir, "$Nest_module.ji") - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) @test last(deps).modpath == ["NestInner"] UsesB_module = :UsesB4b3a94a1a081a8cb @@ -463,7 +478,7 @@ precompile_test_harness(false) do dir """) UsesB = Base.require(Main, UsesB_module) cachefile = joinpath(cachedir, "$UsesB_module.ji") - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) id1, id2 = only(requires) @test Base.pkgorigins[id1].cachepath == cachefile @test Base.pkgorigins[id2].cachepath == joinpath(cachedir, "$B_module.ji") @@ -497,18 +512,19 @@ precompile_test_harness(false) do dir end """) - cachefile = Base.compilecache(Base.PkgId("FooBar")) + cachefile, _ = Base.compilecache(Base.PkgId("FooBar")) empty_prefs_hash = Base.get_preferences_hash(nothing, String[]) @test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), empty_prefs_hash) @test isfile(joinpath(cachedir, "FooBar.ji")) - @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector + Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String} : Tuple{<:Vector, Nothing} + @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @test !isdefined(Main, :FooBar) @test !isdefined(Main, :FooBar1) relFooBar_file = joinpath(dir, "subfolder", "..", "FooBar.jl") - @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Vector : Bool) # `..` is not a symlink on Windows + @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String} : Bool) # `..` is not a symlink on Windows mkdir(joinpath(dir, "subfolder")) - @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector + @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @eval using FooBar fb_uuid = Base.module_build_id(FooBar) @@ -520,7 +536,7 @@ precompile_test_harness(false) do dir @test !isfile(joinpath(cachedir, "FooBar1.ji")) @test isfile(joinpath(cachedir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) === true - @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Vector + @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tsc @test fb_uuid == Base.module_build_id(FooBar) fb_uuid1 = Base.module_build_id(FooBar1) @test fb_uuid != fb_uuid1 @@ -1260,7 +1276,11 @@ end end finally cd(save_cwd) - rm(temp_path, recursive=true) + try + rm(temp_path, recursive=true) + catch err + @show err + end pop!(test_workers) # remove myid rmprocs(test_workers) end @@ -1400,13 +1420,13 @@ precompile_test_harness("Issue #25971") do load_path sourcefile = joinpath(load_path, "Foo25971.jl") write(sourcefile, "module Foo25971 end") chmod(sourcefile, 0o666) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) @test filemode(sourcefile) == filemode(cachefile) chmod(sourcefile, 0o600) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) @test filemode(sourcefile) == filemode(cachefile) chmod(sourcefile, 0o444) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) # Check writable @test touch(cachefile) == cachefile end @@ -1597,6 +1617,80 @@ precompile_test_harness("Module tparams") do load_path @test ModuleTparams.the_struct === Base.invokelatest(ModuleTparams.ParamStruct{ModuleTparams.TheTParam}) end +precompile_test_harness("PkgCacheInspector") do load_path + # Test functionality needed by PkgCacheInspector.jl + write(joinpath(load_path, "PCI.jl"), + """ + module PCI + Base.repl_cmd() = 55 # external method + f() = Base.repl_cmd(7, "hello") # external specialization (should never exist otherwise) + try + f() + catch + end + end + """) + cachefile, ocachefile = Base.compilecache(Base.PkgId("PCI")) + + # Get the depmods + local depmods + @lock Base.require_lock begin + local depmodnames + io = open(cachefile, "r") + try + # isvalid_cache_header returns checksum id or zero + Base.isvalid_cache_header(io) == 0 && throw(ArgumentError("Invalid header in cache file $cachefile.")) + depmodnames = Base.parse_cache_header(io)[3] + Base.isvalid_file_crc(io) || throw(ArgumentError("Invalid checksum in cache file $cachefile.")) + finally + close(io) + end + ndeps = length(depmodnames) + depmods = Vector{Any}(undef, ndeps) + for i in 1:ndeps + modkey, build_id = depmodnames[i] + dep = Base._tryrequire_from_serialized(modkey, build_id) + if !isa(dep, Module) + return dep + end + depmods[i] = dep + end + end + + if ocachefile !== nothing + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint), ocachefile, depmods, true) + else + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), cachefile, depmods, true) + end + + modules, init_order, external_methods, new_specializations, new_method_roots, external_targets, edges = sv + m = only(external_methods) + @test m.name == :repl_cmd && m.nargs < 2 + @test any(new_specializations) do ci + mi = ci.def + mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} + end +end + +precompile_test_harness("DynamicExpressions") do load_path + # https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312 + write(joinpath(load_path, "Float16MWE.jl"), + """ + module Float16MWE + struct Node{T} + val::T + end + doconvert(::Type{<:Node}, val) = convert(Float16, val) + precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64}) + end # module Float16MWE + """) + Base.compilecache(Base.PkgId("Float16MWE")) + (@eval (using Float16MWE)) + Base.invokelatest() do + @test Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2) === Float16(-1.2) + end +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH)