Rebase JuliaLang#40414 and update for Julia v1.9 + add test

base/Base.jl

@@ -500,6 +500,14 @@ function __init__()
     nothing
 end
 
+function __init_build()
+    reinit_stdio()
+    Sys.__init_build()
+    init_depot_path()
+    init_load_path()
+    init_active_project()
+end
+
 # enable threads support
 @eval PCRE PCRE_COMPILE_LOCK = Threads.SpinLock()
 

src/aotcompile.cpp

@@ -59,6 +59,7 @@ using namespace llvm;
 #include "julia_internal.h"
 #include "jitlayers.h"
 #include "julia_assert.h"
+#include "codegen_shared.h"
 
 template<class T> // for GlobalObject's
 static T *addComdat(T *G)
@@ -145,10 +146,11 @@ static void emit_offset_table(Module &mod, const std::vector<GlobalValue*> &vars
         addrs[i] = ConstantExpr::getBitCast(var, T_psize);
     }
     ArrayType *vars_type = ArrayType::get(T_psize, nvars);
-    new GlobalVariable(mod, vars_type, true,
+    GlobalVariable *GV = new GlobalVariable(mod, vars_type, true,
                        GlobalVariable::ExternalLinkage,
                        ConstantArray::get(vars_type, addrs),
                        name);
+    GV->setSection(JL_SYSIMG_LINK_SECTION);
 }
 
 static bool is_safe_char(unsigned char c)
@@ -240,6 +242,32 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance
     *ci_out = codeinst;
 }
 
+StringRef lookup_sysimage_fname(void *ptr, jl_code_instance_t *codeinst)
+{
+    if (ptr == (void*)&jl_fptr_args_addr) {
+        return "jl_fptr_args";
+    } else if (ptr == (void*)&jl_fptr_sparam_addr) {
+        return "jl_fptr_sparam";
+    } else if (ptr == (void*)&jl_fptr_const_return_addr) {
+        return "jl_fptr_const_return";
+    }
+    return jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)ptr, codeinst, false);
+}
+
+static void add_gv(void *ctx, void *mod, jl_value_t **gv_slot)
+{
+    jl_codegen_params_t *params = (jl_codegen_params_t*)ctx;
+    Module *M = (Module *)mod;
+    GlobalVariable* &lgv = params->globals[*gv_slot];
+    if (!lgv){
+        lgv = new GlobalVariable(*M,
+            JuliaType::get_pjlvalue_ty(M->getContext()),
+            false, GlobalVariable::PrivateLinkage,
+            NULL, jl_ExecutionEngine->getGlobalAtAddress((uintptr_t)gv_slot));
+        lgv->setExternallyInitialized(true);
+    }
+}
+
 // takes the running content that has collected in the shadow module and dump it to disk
 // this builds the object file portion of the sysimage files for fast startup, and can
 // also be used be extern consumers like GPUCompiler.jl to obtain a module containing
@@ -275,6 +303,11 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
 
     params.imaging = imaging;
 
+    bool sysimg_chained = jl_options.use_sysimage_native_code == JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_CHAINED;
+    if (sysimg_chained) {
+        jl_foreach_sysimg_gvar_slot(add_gv, (void*)&params, (void*)clone.getModuleUnlocked());
+    }
+
     // compile all methods for the current world and type-inference world
     size_t compile_for[] = { jl_typeinf_world, jl_atomic_load_acquire(&jl_world_counter) };
     for (int worlds = 0; worlds < 2; worlds++) {
@@ -303,10 +336,33 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
                 // find and prepare the source code to compile
                 jl_code_instance_t *codeinst = NULL;
                 jl_ci_cache_lookup(*cgparams, mi, params.world, &codeinst, &src);
-                if (src && !emitted.count(codeinst)) {
+                // determines if the instance should be compiled
+                bool compile = true;
+                uint8_t precompile = jl_atomic_load_relaxed(&codeinst->precompile);
+                if (codeinst && (precompile & 2)) {
+                    // This condition (precompile & 2) provides a speed-up
+                    // Skip things already in the sysimage, we'll pick it up from there.
+                    jl_llvm_functions_t fnames = {
+                        lookup_sysimage_fname((void*)(codeinst->invoke.load()), codeinst).str(),
+                        lookup_sysimage_fname(codeinst->specptr.fptr, codeinst).str(),
+                    };
+                    if (!fnames.functionObject.empty() && !fnames.specFunctionObject.empty()){
+                        orc::ThreadSafeModule no_module;
+                        if (emitted.find(codeinst) == emitted.end()){
+                            emitted[codeinst] = {std::move(no_module), std::move(fnames)};
+                        }
+                        compile = false;
+                    }
+                }
+                if (sysimg_chained && !(precompile & 4)){
+                    // not tagged for being compiled to sysimage
+                    compile = false;
+                }
+                if (compile && src && !emitted.count(codeinst)) {
                     // now add it to our compilation results
                     JL_GC_PROMISE_ROOTED(codeinst->rettype);
-                    orc::ThreadSafeModule result_m = jl_create_llvm_module(name_from_method_instance(codeinst->def),
+                    orc::ThreadSafeModule result_m = jl_create_llvm_module(
+                            name_from_method_instance(codeinst->def),
                             params.tsctx, params.imaging,
                             clone.getModuleUnlocked()->getDataLayout(),
                             Triple(clone.getModuleUnlocked()->getTargetTriple()));
@@ -333,11 +389,30 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
     // clones the contents of the module `m` to the shadow_output collector
     // while examining and recording what kind of function pointer we have
     for (auto &def : emitted) {
-        jl_merge_module(clone, std::move(std::get<0>(def.second)));
         jl_code_instance_t *this_code = def.first;
         jl_llvm_functions_t decls = std::get<1>(def.second);
         StringRef func = decls.functionObject;
         StringRef cfunc = decls.specFunctionObject;
+        if (std::get<0>(def.second))
+            jl_merge_module(clone, std::move(std::get<0>(def.second)));
+        else {
+            // TODO: Probably wait until all other modules were merged
+            // TODO: These signatures aren't actually right, but it's not worth
+            // trying to compute signatures for these. Maybe declare them as
+            // void* global variables instead and have jl_merge_module know
+            // how to merge them if it comes to it?
+            auto &context = clone.getModuleUnlocked()->getContext();
+            FunctionType *jl_func_sig = JuliaType::get_jlfunc_ty(context);
+
+            Function::Create(jl_func_sig,
+                GlobalVariable::ExternalLinkage,
+                func, clone.getModuleUnlocked());
+            if (!cfunc.empty()) {
+                Function::Create(jl_func_sig,
+                    GlobalVariable::ExternalLinkage,
+                    cfunc, clone.getModuleUnlocked());
+            }
+        }
         uint32_t func_id = 0;
         uint32_t cfunc_id = 0;
         if (func == "jl_fptr_args") {
@@ -346,6 +421,9 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
         else if (func == "jl_fptr_sparam") {
             func_id = -2;
         }
+        else if (func == "jl_fptr_const_return") {
+            func_id = -3;
+        }
         else {
             //Safe b/c context is locked by params
             data->jl_sysimg_fvars.push_back(cast<Function>(clone.getModuleUnlocked()->getNamedValue(func)));
@@ -365,10 +443,12 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
     // now get references to the globals in the merged module
     // and set them to be internalized and initialized at startup
     for (auto &global : gvars) {
-        //Safe b/c context is locked by params
-        GlobalVariable *G = cast<GlobalVariable>(clone.getModuleUnlocked()->getNamedValue(global));
-        G->setInitializer(ConstantPointerNull::get(cast<PointerType>(G->getValueType())));
-        G->setLinkage(GlobalVariable::InternalLinkage);
+        auto gv = clone.getModuleUnlocked()->getNamedValue(global);
+        GlobalVariable *G = cast<GlobalVariable>(gv);
+        if (!G->isExternallyInitialized())
+            G->setInitializer(ConstantPointerNull::get(cast<PointerType>(G->getValueType())));
+        G->setLinkage(GlobalVariable::ExternalLinkage);
+        G->setVisibility(GlobalVariable::HiddenVisibility);
         data->jl_sysimg_gvars.push_back(G);
     }
 
@@ -389,7 +469,10 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
         //Safe b/c context is locked by params
         for (GlobalObject &G : clone.getModuleUnlocked()->global_objects()) {
             if (!G.isDeclaration()) {
-                G.setLinkage(Function::InternalLinkage);
+                if (G.getLinkage() != GlobalVariable::PrivateLinkage) {
+                    G.setLinkage(Function::ExternalLinkage);
+                    G.setVisibility(GlobalVariable::HiddenVisibility);
+                }
                 makeSafeName(G);
                 addComdat(&G);
 #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_)
@@ -496,6 +579,8 @@ void jl_dump_native_impl(void *native_code,
     std::vector<NewArchiveMember> unopt_bc_Archive;
     std::vector<std::string> outputs;
 
+    bool sysimg_chained = jl_options.use_sysimage_native_code == JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_CHAINED;
+
     legacy::PassManager preopt, postopt;
 
     if (unopt_bc_fname)
@@ -515,7 +600,7 @@ void jl_dump_native_impl(void *native_code,
     legacy::PassManager optimizer;
     if (bc_fname || obj_fname || asm_fname) {
         addTargetPasses(&optimizer, TM->getTargetTriple(), TM->getTargetIRAnalysis());
-        addOptimizationPasses(&optimizer, jl_options.opt_level, true, true);
+        addOptimizationPasses(&optimizer, jl_options.opt_level, true, true, false, sysimg_chained);
         addMachinePasses(&optimizer, jl_options.opt_level);
     }
 
@@ -538,12 +623,15 @@ void jl_dump_native_impl(void *native_code,
         // reflect the address of the jl_RTLD_DEFAULT_handle variable
         // back to the caller, so that we can check for consistency issues
         GlobalValue *jlRTLD_DEFAULT_var = jl_emit_RTLD_DEFAULT_var(dataM);
-        addComdat(new GlobalVariable(*dataM,
+        GlobalVariable *jlRTLD_DEFAULT_var_pointer =
+            new GlobalVariable(*dataM,
                                      jlRTLD_DEFAULT_var->getType(),
                                      true,
                                      GlobalVariable::ExternalLinkage,
                                      jlRTLD_DEFAULT_var,
-                                     "jl_RTLD_DEFAULT_handle_pointer"));
+                                     "jl_RTLD_DEFAULT_handle_pointer");
+        jlRTLD_DEFAULT_var_pointer->setSection(JL_SYSIMG_LINK_SECTION);
+        addComdat(jlRTLD_DEFAULT_var_pointer);
     }
 
     // do the actual work
@@ -622,7 +710,7 @@ void addMachinePasses(legacy::PassManagerBase *PM, int optlevel)
 // it assumes that the TLI and TTI wrapper passes have already been added.
 void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
                            bool lower_intrinsics, bool dump_native,
-                           bool external_use)
+                           bool external_use, bool chained)
 {
     // Note: LLVM 12 disabled the hoisting of common instruction
     //       before loop vectorization (https://reviews.llvm.org/D84108).
@@ -682,7 +770,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
             PM->add(createRemoveNIPass());
         }
         PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop
-        if (dump_native) {
+        if (dump_native && !chained) {
             PM->add(createMultiVersioningPass(external_use));
             PM->add(createCPUFeaturesPass());
             // minimal clean-up to get rid of CPU feature checks
@@ -724,7 +812,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
     // consider AggressiveInstCombinePass at optlevel > 2
     PM->add(createInstructionCombiningPass());
     PM->add(createCFGSimplificationPass(basicSimplifyCFGOptions));
-    if (dump_native)
+    if (dump_native && !chained)
         PM->add(createMultiVersioningPass(external_use));
     PM->add(createCPUFeaturesPass());
     PM->add(createSROAPass());

src/ccall.cpp

@@ -48,7 +48,9 @@ STATISTIC(SRetCCalls, "Number of ccalls that were marked sret");
 // somewhat unusual variable, in that aotcompile wants to get the address of this for a sanity check
 GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M)
 {
-    return prepare_global_in(M, jlRTLD_DEFAULT_var);
+    GlobalVariable *var = prepare_global_in(M, jlRTLD_DEFAULT_var);
+    var->setSection(JL_SYSIMG_LINK_SECTION);
+    return var;
 }
 
 

src/cgutils.cpp

@@ -119,6 +119,7 @@ static Value *stringConstPtr(
     StringRef ctxt(txt.c_str(), txt.size() + 1);
     Constant *Data = ConstantDataArray::get(irbuilder.getContext(), arrayRefFromStringRef(ctxt));
     GlobalVariable *gv = get_pointer_to_constant(emission_context, Data, "_j_str", *M);
+    gv->setLinkage(GlobalVariable::PrivateLinkage);
     Value *zero = ConstantInt::get(Type::getInt32Ty(irbuilder.getContext()), 0);
     Value *Args[] = { zero, zero };
     return irbuilder.CreateInBoundsGEP(gv->getValueType(), gv, Args);
@@ -301,16 +302,26 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr)
     if (!gv) {
         raw_string_ostream(gvname) << cname << ctx.global_targets.size();
         localname = StringRef(gvname);
+        gv = new GlobalVariable(*M, ctx.types().T_pjlvalue,
+                                false, GlobalVariable::InternalLinkage,
+                                NULL, localname);
     }
     else {
         localname = gv->getName();
-        if (gv->getParent() != M)
+        if (gv->getParent() != M) {
+            GlobalVariable *oldgv = gv;
             gv = cast_or_null<GlobalVariable>(M->getNamedValue(localname));
+            if (!gv) {
+                gv = new GlobalVariable(*M, ctx.types().T_pjlvalue,
+                                        false, oldgv->getLinkage(),
+                                        NULL, localname);
+                if (oldgv->isExternallyInitialized()) {
+                    gv->setExternallyInitialized(true);
+                }
+            }
+        }
     }
-    if (gv == nullptr)
-        gv = new GlobalVariable(*M, ctx.types().T_pjlvalue,
-                                false, GlobalVariable::PrivateLinkage,
-                                NULL, localname);
+    assert(gv != nullptr);
     // LLVM passes sometimes strip metadata when moving load around
     // since the load at the new location satisfy the same condition as the original one.
     // Mark the global as constant to LLVM code using our own metadata
@@ -390,7 +401,7 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p)
         return julia_pgv(ctx, "jl_sym#", addr, NULL, p);
     }
     // something else gets just a generic name
-    return julia_pgv(ctx, "jl_global#", p);
+    return julia_pgv(ctx, "jl_global#abc#", p);
 }
 
 static size_t dereferenceable_size(jl_value_t *jt)

src/codegen-stubs.c

@@ -68,6 +68,10 @@ JL_DLLEXPORT size_t jl_jit_total_bytes_fallback(void)
 
 JL_DLLEXPORT void *jl_create_native_fallback(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmctxt, const jl_cgparams_t *cgparams, int _policy) UNAVAILABLE
 
+JL_DLLEXPORT void jl_init_sysimage_chaining_fallback(void *sysimg_base, const char *fname)
+{
+}
+
 JL_DLLEXPORT void jl_dump_compiles_fallback(void *s)
 {
 }

src/codegen.cpp

@@ -1535,11 +1535,16 @@ static inline GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G)
 
 static GlobalVariable *get_pointer_to_constant(jl_codegen_params_t &emission_context, Constant *val, StringRef name, Module &M)
 {
+    bool sysimg_chained = jl_options.use_sysimage_native_code == JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_CHAINED;
     GlobalVariable *&gv = emission_context.mergedConstants[val];
     StringRef localname;
     std::string ssno;
     if (gv == nullptr) {
         raw_string_ostream(ssno) << name << emission_context.mergedConstants.size();
+        if (sysimg_chained){
+            // modify the name for sysimage to prevent collisions
+            raw_string_ostream(ssno) << "chained";
+        }
         localname = StringRef(ssno);
     }
     else {
@@ -1663,7 +1668,9 @@ static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, Value *v, jl_value_
 {
     Value *loc;
     if (valid_as_globalinit(v)) { // llvm can't handle all the things that could be inside a ConstantExpr
-        loc = get_pointer_to_constant(ctx.emission_context, cast<Constant>(v), "_j_const", *jl_Module);
+        GlobalVariable *gv = get_pointer_to_constant(ctx.emission_context, cast<Constant>(v), "_j_const", *jl_Module);
+        gv->setLinkage(GlobalVariable::PrivateLinkage);
+        loc = gv;
     }
     else {
         loc = emit_static_alloca(ctx, v->getType());

src/gf.c

@@ -214,6 +214,13 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ
     return sf->func.value;
 }
 
+uint8_t precompiles_for_sysimage = 0;
+
+JL_DLLEXPORT void jl_precompiles_for_sysimage(uint8_t enable)
+{
+    precompiles_for_sysimage = enable;
+}
+
 // ----- MethodInstance specialization instantiation ----- //
 
 JL_DLLEXPORT jl_code_instance_t* jl_new_codeinst(
@@ -2136,7 +2143,8 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t
     else {
         record_precompile_statement(mi);
     }
-    jl_atomic_store_relaxed(&codeinst->precompile, 1);
+    uint8_t precompile = precompiles_for_sysimage ? 5 : 1;
+    jl_atomic_store_relaxed(&codeinst->precompile, precompile);
     return codeinst;
 }
 
@@ -2264,7 +2272,8 @@ static void _generate_from_hint(jl_method_instance_t *mi, size_t world)
     if (codeinst != jl_nothing) {
         if (jl_atomic_load_relaxed(&((jl_code_instance_t*)codeinst)->invoke) == jl_fptr_const_return)
             return; // probably not a good idea to generate code
-        jl_atomic_store_relaxed(&((jl_code_instance_t*)codeinst)->precompile, 1);
+        uint8_t precompile = precompiles_for_sysimage ? 5 : 1;
+        jl_atomic_store_relaxed(&((jl_code_instance_t*)codeinst)->precompile, precompile);
     }
 }