From 992c5203deb2ef9fe7beff85e7f90ee4874879a9 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Tue, 18 Feb 2025 19:25:13 -0500 Subject: [PATCH 01/25] Use `Base.process_status(p)` instead of `p.exitcode` in error message (#57460) Fixup https://github.com/JuliaLang/julia/pull/57455 --- base/loading.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index 6365d2bbdbf6dc..0f99cceab3045d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -3288,7 +3288,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in if p.exitcode == 125 return PrecompilableError() else - error("Failed to precompile $(repr("text/plain", pkg)) to $(repr(tmppath)) (exit code $(p.exitcode)).") + error("Failed to precompile $(repr("text/plain", pkg)) to $(repr(tmppath)) ($(Base.process_status(p))).") end end From a65c2cfb2b9b9e4ea6df9988ad9a54366e6236c7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 19 Feb 2025 10:30:32 -0500 Subject: [PATCH 02/25] Revert "Make emitted egal code more loopy (#54121)" (#57453) This reverts a portion of commit 50833c84d454ef989797e035294ba27b3cca79b7. This algorithm is not able to handle simple cases where there is any internal padding, such as the example of: ``` struct LotsBytes a::Int8 b::NTuple{256,Int} c::Int end ``` Unfortunately fixing it is a bit of a large project right now, so reverting now to fix correctness while working on that. Fixes #55513 (indirectly, by removing broken code) Maybe reopens #54109, although the latency issue it proposes to fix doesn't occur on master even with this revert (just the mediocre looking IR result output returns) --- Compiler/test/codegen.jl | 51 -------------- src/codegen.cpp | 141 --------------------------------------- 2 files changed, 192 deletions(-) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 57a9c26aefac67..4514852a2c0bc8 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -889,57 +889,6 @@ ex54166 = Union{Missing, Int64}[missing -2; missing -2]; dims54166 = (1,2) @test (minimum(ex54166; dims=dims54166)[1] === missing) -# #54109 - Excessive LLVM time for egal -struct DefaultOr54109{T} - x::T - default::Bool -end - -@eval struct Torture1_54109 - $((Expr(:(::), Symbol("x$i"), DefaultOr54109{Float64}) for i = 1:897)...) -end -Torture1_54109() = Torture1_54109((DefaultOr54109(1.0, false) for i = 1:897)...) - -@eval struct Torture2_54109 - $((Expr(:(::), Symbol("x$i"), DefaultOr54109{Float64}) for i = 1:400)...) - $((Expr(:(::), Symbol("x$(i+400)"), DefaultOr54109{Int16}) for i = 1:400)...) -end -Torture2_54109() = Torture2_54109((DefaultOr54109(1.0, false) for i = 1:400)..., (DefaultOr54109(Int16(1), false) for i = 1:400)...) - -@noinline egal_any54109(x, @nospecialize(y::Any)) = x === Base.compilerbarrier(:type, y) - -let ir1 = get_llvm(egal_any54109, Tuple{Torture1_54109, Any}), - ir2 = get_llvm(egal_any54109, Tuple{Torture2_54109, Any}) - - # We can't really do timing on CI, so instead, let's look at the length of - # the optimized IR. The original version had tens of thousands of lines and - # was slower, so just check here that we only have < 500 lines. If somebody, - # implements a better comparison that's larger than that, just re-benchmark - # this and adjust the threshold. - - @test count(==('\n'), ir1) < 500 - @test count(==('\n'), ir2) < 500 -end - -## Regression test for egal of a struct of this size without padding, but with -## non-bitsegal, to make sure that it doesn't accidentally go down the accelerated -## path. -@eval struct BigStructAnyInt - $((Expr(:(::), Symbol("x$i"), Pair{Any, Int}) for i = 1:33)...) -end -BigStructAnyInt() = BigStructAnyInt((Union{Base.inferencebarrier(Float64), Int}=>i for i = 1:33)...) -@test egal_any54109(BigStructAnyInt(), BigStructAnyInt()) - -## For completeness, also test correctness, since we don't have a lot of -## large-struct tests. - -# The two allocations of the same struct will likely have different padding, -# we want to make sure we find them egal anyway - a naive memcmp would -# accidentally look at it. -@test egal_any54109(Torture1_54109(), Torture1_54109()) -@test egal_any54109(Torture2_54109(), Torture2_54109()) -@test !egal_any54109(Torture1_54109(), Torture1_54109((DefaultOr54109(2.0, false) for i = 1:897)...)) - bar54599() = Base.inferencebarrier(true) ? (Base.PkgId(Main),1) : nothing function foo54599() diff --git a/src/codegen.cpp b/src/codegen.cpp index fe2bd4fc3d0972..ac74e1443685a3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3616,61 +3616,6 @@ static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, return phi; } -struct egal_desc { - size_t offset; - size_t nrepeats; - size_t data_bytes; - size_t padding_bytes; -}; - -template -static size_t emit_masked_bits_compare(callback &emit_desc, jl_datatype_t *aty, egal_desc ¤t_desc) -{ - // Memcmp, but with masked padding - size_t data_bytes = 0; - size_t padding_bytes = 0; - size_t nfields = jl_datatype_nfields(aty); - size_t total_size = jl_datatype_size(aty); - assert(aty->layout->flags.isbitsegal); - for (size_t i = 0; i < nfields; ++i) { - size_t offset = jl_field_offset(aty, i); - size_t fend = i == nfields - 1 ? total_size : jl_field_offset(aty, i + 1); - size_t fsz = jl_field_size(aty, i); - jl_datatype_t *fty = (jl_datatype_t*)jl_field_type(aty, i); - assert(jl_is_datatype(fty)); // union fields should never reach here - assert(fty->layout->flags.isbitsegal); - if (jl_field_isptr(aty, i) || !fty->layout->flags.haspadding) { - // The field has no internal padding - data_bytes += fsz; - if (offset + fsz == fend) { - // The field has no padding after. Merge this into the current - // comparison range and go to next field. - } else { - padding_bytes = fend - offset - fsz; - // Found padding. Either merge this into the current comparison - // range, or emit the old one and start a new one. - if (current_desc.data_bytes == data_bytes && - current_desc.padding_bytes == padding_bytes) { - // Same as the previous range, just note that down, so we - // emit this as a loop. - current_desc.nrepeats += 1; - } else { - if (current_desc.nrepeats != 0) - emit_desc(current_desc); - current_desc.nrepeats = 1; - current_desc.data_bytes = data_bytes; - current_desc.padding_bytes = padding_bytes; - } - data_bytes = 0; - } - } else { - // The field may have internal padding. Recurse this. - data_bytes += emit_masked_bits_compare(emit_desc, fty, current_desc); - } - } - return data_bytes; -} - static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t arg2) { ++EmittedBitsCompares; @@ -3747,92 +3692,6 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a } return ctx.builder.CreateICmpEQ(answer, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); } - else if (sz > 512 && jl_struct_try_layout(sty) && sty->layout->flags.isbitsegal) { - Value *varg1 = arg1.inline_roots.empty() && arg1.ispointer() ? data_pointer(ctx, arg1) : - value_to_pointer(ctx, arg1).V; - Value *varg2 = arg2.inline_roots.empty() && arg2.ispointer() ? data_pointer(ctx, arg2) : - value_to_pointer(ctx, arg2).V; - varg1 = emit_pointer_from_objref(ctx, varg1); - varg2 = emit_pointer_from_objref(ctx, varg2); - - // See above for why we want to do this - SmallVector gc_uses; - gc_uses.append(get_gc_roots_for(ctx, arg1)); - gc_uses.append(get_gc_roots_for(ctx, arg2)); - OperandBundleDef OpBundle("jl_roots", gc_uses); - - Value *answer = nullptr; - auto emit_desc = [&](egal_desc desc) { - Value *ptr1 = varg1; - Value *ptr2 = varg2; - if (desc.offset != 0) { - ptr1 = emit_ptrgep(ctx, ptr1, desc.offset); - ptr2 = emit_ptrgep(ctx, ptr2, desc.offset); - } - - Value *new_ptr1 = ptr1; - Value *endptr1 = nullptr; - BasicBlock *postBB = nullptr; - BasicBlock *loopBB = nullptr; - PHINode *answerphi = nullptr; - if (desc.nrepeats != 1) { - // Set up loop - endptr1 = emit_ptrgep(ctx, ptr1, desc.nrepeats * (desc.data_bytes + desc.padding_bytes));; - - BasicBlock *currBB = ctx.builder.GetInsertBlock(); - loopBB = BasicBlock::Create(ctx.builder.getContext(), "egal_loop", ctx.f); - postBB = BasicBlock::Create(ctx.builder.getContext(), "post", ctx.f); - ctx.builder.CreateBr(loopBB); - - ctx.builder.SetInsertPoint(loopBB); - Type *TInt1 = getInt1Ty(ctx.builder.getContext()); - answerphi = ctx.builder.CreatePHI(TInt1, 2); - answerphi->addIncoming(answer ? answer : ConstantInt::get(TInt1, 1), currBB); - answer = answerphi; - - PHINode *itr1 = ctx.builder.CreatePHI(ptr1->getType(), 2); - PHINode *itr2 = ctx.builder.CreatePHI(ptr2->getType(), 2); - - new_ptr1 = emit_ptrgep(ctx, itr1, desc.data_bytes + desc.padding_bytes); - itr1->addIncoming(ptr1, currBB); - itr1->addIncoming(new_ptr1, loopBB); - - Value *new_ptr2 = emit_ptrgep(ctx, itr2, desc.data_bytes + desc.padding_bytes); - itr2->addIncoming(ptr2, currBB); - itr2->addIncoming(new_ptr2, loopBB); - - ptr1 = itr1; - ptr2 = itr2; - } - - // Emit memcmp. TODO: LLVM has a pass to expand this for additional - // performance. - Value *this_answer = ctx.builder.CreateCall(prepare_call(memcmp_func), - { ptr1, - ptr2, - ConstantInt::get(ctx.types().T_size, desc.data_bytes) }, - ArrayRef(&OpBundle, gc_uses.empty() ? 0 : 1)); - this_answer = ctx.builder.CreateICmpEQ(this_answer, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); - answer = answer ? ctx.builder.CreateAnd(answer, this_answer) : this_answer; - if (endptr1) { - answerphi->addIncoming(answer, loopBB); - Value *loopend = ctx.builder.CreateICmpEQ(new_ptr1, endptr1); - ctx.builder.CreateCondBr(loopend, postBB, loopBB); - ctx.builder.SetInsertPoint(postBB); - } - }; - egal_desc current_desc = {0}; - size_t trailing_data_bytes = emit_masked_bits_compare(emit_desc, sty, current_desc); - assert(current_desc.nrepeats != 0); - emit_desc(current_desc); - if (trailing_data_bytes != 0) { - current_desc.nrepeats = 1; - current_desc.data_bytes = trailing_data_bytes; - current_desc.padding_bytes = 0; - emit_desc(current_desc); - } - return answer; - } else { jl_svec_t *types = sty->types; Value *answer = ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); From b9a8d46a234fa0fe625371e107f3ab441a867182 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 19 Feb 2025 12:58:11 -0300 Subject: [PATCH 03/25] Change memory indexing to use the type as index instead of i8 (#57389) also add nsw/nuw flags whenever possible. --- src/cgutils.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 39879503596fed..eeba59aba3aede 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4710,10 +4710,9 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg setName(ctx.emission_context, ovflw, "memoryref_ovflw"); } #endif - boffset = ctx.builder.CreateMul(offset, elsz); - setName(ctx.emission_context, boffset, "memoryref_byteoffset"); - newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), data, boffset); - setName(ctx.emission_context, newdata, "memoryref_data_byteoffset"); + Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(ref.typ)); + newdata = ctx.builder.CreateGEP(elty, data, offset); + setName(ctx.emission_context, newdata, "memoryref_data_offset"); (void)boffset; // LLVM is very bad at handling GEP with types different from the load if (bc) { BasicBlock *failBB, *endBB; From e67a22cec85e9126104b38a4c34097b93955132d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20G=C3=B6ttgens?= Date: Wed, 19 Feb 2025 20:59:40 +0100 Subject: [PATCH 04/25] Convert some julia-repl examples to doctests (#57452) Some progress towards https://github.com/JuliaLang/julia/issues/56921. --- base/bitarray.jl | 2 +- base/essentials.jl | 4 ++-- base/expr.jl | 8 ++++---- base/loading.jl | 2 +- base/twiceprecision.jl | 6 +++--- doc/src/manual/arrays.md | 10 +++++----- doc/src/manual/functions.md | 21 +++++++++++++-------- doc/src/manual/metaprogramming.md | 2 +- stdlib/Sockets/src/addrinfo.jl | 2 +- 9 files changed, 31 insertions(+), 26 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index 93fa48c56e3798..5c29c2a107cf00 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -81,7 +81,7 @@ BitVector() = BitVector(undef, 0) Construct a `BitVector` from a tuple of `Bool`. # Examples -```julia-repl +```jldoctest julia> nt = (true, false, true, false) (true, false, true, false) diff --git a/base/essentials.jl b/base/essentials.jl index 5db7a5f6fb0d94..795e4998895829 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -93,7 +93,7 @@ f(y) = [x for x in y] # Examples -```julia-repl +```jldoctest; setup = :(using InteractiveUtils) julia> f(A::AbstractArray) = g(A) f (generic function with 1 method) @@ -102,7 +102,7 @@ g (generic function with 1 method) julia> @code_typed f([1.0]) CodeInfo( -1 ─ %1 = invoke Main.g(_2::AbstractArray)::Float64 +1 ─ %1 = invoke g(A::AbstractArray)::Float64 └── return %1 ) => Float64 ``` diff --git a/base/expr.jl b/base/expr.jl index 8ae394e1224438..913d4b690f9181 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -103,7 +103,7 @@ Take the expression `x` and return an equivalent expression with all macros remo for executing in module `m`. The `recursive` keyword controls whether deeper levels of nested macros are also expanded. This is demonstrated in the example below: -```julia-repl +```jldoctest; filter = r"#= .*:6 =#" julia> module M macro m1() 42 @@ -118,7 +118,7 @@ julia> macroexpand(M, :(@m2()), recursive=true) 42 julia> macroexpand(M, :(@m2()), recursive=false) -:(#= REPL[16]:6 =# M.@m1) +:(#= REPL[1]:6 =# @m1) ``` """ function macroexpand(m::Module, @nospecialize(x); recursive=true) @@ -926,7 +926,7 @@ This can be used to limit the number of compiler-generated specializations durin # Examples -```julia +```jldoctest; setup = :(using InteractiveUtils) julia> f(A::AbstractArray) = g(A) f (generic function with 1 method) @@ -935,7 +935,7 @@ g (generic function with 1 method) julia> @code_typed f([1.0]) CodeInfo( -1 ─ %1 = invoke Main.g(_2::AbstractArray)::Any +1 ─ %1 = invoke g(A::AbstractArray)::Any └── return %1 ) => Any ``` diff --git a/base/loading.jl b/base/loading.jl index 0f99cceab3045d..197f6eb2b625be 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -417,7 +417,7 @@ otherwise it searches all recursive dependencies (from the resolved manifest of each environment) until it locates the context `where`, and from there identifies the dependency with the corresponding name. -```julia-repl +```jldoctest julia> Base.identify_package("Pkg") # Pkg is a dependency of the default environment Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] diff --git a/base/twiceprecision.jl b/base/twiceprecision.jl index 6928d420a3860b..920ba71eba24ff 100644 --- a/base/twiceprecision.jl +++ b/base/twiceprecision.jl @@ -63,7 +63,7 @@ representation, even though it is exact from the standpoint of binary representation. Example: -```julia-repl +```jldoctest julia> 1.0 + 1.0001e-15 1.000000000000001 @@ -94,7 +94,7 @@ numbers. Mathematically, `zhi + zlo = x * y`, where `zhi` contains the most significant bits and `zlo` the least significant. Example: -```julia-repl +```jldoctest julia> x = Float32(π) 3.1415927f0 @@ -126,7 +126,7 @@ numbers. Mathematically, `zhi + zlo ≈ x / y`, where `zhi` contains the most significant bits and `zlo` the least significant. Example: -```julia-repl +```jldoctest julia> x, y = Float32(π), 3.1f0 (3.1415927f0, 3.1f0) diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index 02d71fcd9939e3..ee9421ba1730b9 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -1160,7 +1160,7 @@ dimension `d`. For example, the builtin `Array` returned by `rand(5,7,2)` has it arranged contiguously in column major order. This means that the stride of the first dimension — the spacing between elements in the same column — is `1`: -```julia-repl +```jldoctest strides julia> A = rand(5, 7, 2); julia> stride(A, 1) @@ -1172,7 +1172,7 @@ as many elements as there are in a single column (`5`). Similarly, jumping betwe "pages" (in the third dimension) requires skipping `5*7 == 35` elements. The [`strides`](@ref) of this array is the tuple of these three numbers together: -```julia-repl +```jldoctest strides julia> strides(A) (1, 5, 35) ``` @@ -1185,7 +1185,7 @@ This view `V` refers to the same memory as `A` but is skipping and re-arranging elements. The stride of the first dimension of `V` is `3` because we're only selecting every third row from our original array: -```julia-repl +```jldoctest strides julia> V = @view A[1:3:4, 2:2:6, 2:-1:1]; julia> stride(V, 1) @@ -1196,7 +1196,7 @@ This view is similarly selecting every other column from our original `A` — an needs to skip the equivalent of two five-element columns when moving between indices in the second dimension: -```julia-repl +```jldoctest strides julia> stride(V, 2) 10 ``` @@ -1205,7 +1205,7 @@ The third dimension is interesting because its order is reversed! Thus to get fr "page" to the second one it must go _backwards_ in memory, and so its stride in this dimension is negative! -```julia-repl +```jldoctest strides julia> stride(V, 3) -35 ``` diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index 0fcfdeb80d7b9a..fd97b44cdc3452 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -584,10 +584,12 @@ The destructuring feature can also be used within a function argument. If a function argument name is written as a tuple (e.g. `(x, y)`) instead of just a symbol, then an assignment `(x, y) = argument` will be inserted for you: -```julia-repl +```jldoctest julia> minmax(x, y) = (y < x) ? (y, x) : (x, y) +minmax (generic function with 1 method) julia> gap((min, max)) = max - min +gap (generic function with 1 method) julia> gap(minmax(10, 2)) 8 @@ -598,7 +600,7 @@ would be a two-argument function, and this example would not work. Similarly, property destructuring can also be used for function arguments: -```julia-repl +```jldoctest julia> foo((; x, y)) = x + y foo (generic function with 1 method) @@ -616,7 +618,7 @@ julia> foo(A(3, 4)) For anonymous functions, destructuring a single argument requires an extra comma: -```julia-repl +```jldoctest julia> map(((x, y),) -> x + y, [(1, 2), (3, 4)]) 2-element Vector{Int64}: 3 @@ -784,12 +786,15 @@ Optional arguments are actually just a convenient syntax for writing multiple me with different numbers of arguments (see [Note on Optional and keyword Arguments](@ref)). This can be checked for our `date` function example by calling the `methods` function: -```julia-repl +```jldoctest date_default_args; filter = r"@ .*"a julia> methods(date) -# 3 methods for generic function "date": -[1] date(y::Int64) in Main at REPL[1]:1 -[2] date(y::Int64, m::Int64) in Main at REPL[1]:1 -[3] date(y::Int64, m::Int64, d::Int64) in Main at REPL[1]:1 +# 3 methods for generic function "date" from Main: + [1] date(y::Int64, m::Int64, d::Int64) + @ REPL[2]:1 + [2] date(y::Int64, m::Int64) + @ REPL[2]:1 + [3] date(y::Int64) + @ REPL[2]:1 ``` ## Keyword Arguments diff --git a/doc/src/manual/metaprogramming.md b/doc/src/manual/metaprogramming.md index b619021fcef92a..9ed579594571ab 100644 --- a/doc/src/manual/metaprogramming.md +++ b/doc/src/manual/metaprogramming.md @@ -413,7 +413,7 @@ Since expressions are just `Expr` objects which can be constructed programmatica it is possible to dynamically generate arbitrary code which can then be run using [`eval`](@ref). Here is a simple example: -```julia-repl +```jldoctest julia> a = 1; julia> ex = Expr(:call, :+, a, :b) diff --git a/stdlib/Sockets/src/addrinfo.jl b/stdlib/Sockets/src/addrinfo.jl index f5599b8623a0b2..db03cdb7c1dc86 100644 --- a/stdlib/Sockets/src/addrinfo.jl +++ b/stdlib/Sockets/src/addrinfo.jl @@ -129,7 +129,7 @@ Uses the operating system's underlying getaddrinfo implementation, which may do a DNS lookup. # Examples -```julia-repl +```jldoctest julia> getaddrinfo("localhost", IPv6) ip"::1" From a70818c3a4b6e660140c0685e70a8094e095397f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 19 Feb 2025 16:39:38 -0500 Subject: [PATCH 05/25] using/import: ensure world update after each observable operation (#57467) Fix #57316 --- src/julia.h | 8 ++++---- src/module.c | 22 ++++++++++----------- src/toplevel.c | 52 ++++++++++++++++++++++++-------------------------- test/worlds.jl | 10 ++++++++++ 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/src/julia.h b/src/julia.h index 6acd82fdd34016..3206b4f1f033be 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2077,10 +2077,10 @@ JL_DLLEXPORT jl_value_t *jl_checked_assignonce(jl_binding_t *b, jl_module_t *mod JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind); JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from); -JL_DLLEXPORT void jl_module_use(jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_use_as(jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); -JL_DLLEXPORT void jl_module_import(jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_import_as(jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); +JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); +JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); +JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported); JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var); diff --git a/src/module.c b/src/module.c index 50d4d90c7f4bb8..848ced7058af11 100644 --- a/src/module.c +++ b/src/module.c @@ -888,11 +888,11 @@ JL_DLLEXPORT void check_safe_import_from(jl_module_t *m) } // NOTE: we use explici since explicit is a C++ keyword -static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) +static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) { check_safe_import_from(from); jl_binding_t *b = jl_get_binding(from, s); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); if (b->deprecated) { if (jl_get_binding_value(b) == jl_nothing) { // silently skip importing deprecated values assigned to nothing (to allow later mutation) @@ -915,7 +915,7 @@ static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_binding_t *ownerb = b; jl_binding_partition_t *ownerbpart = bpart; - jl_walk_binding_inplace(&ownerb, &ownerbpart, jl_current_task->world_age); + jl_walk_binding_inplace(&ownerb, &ownerbpart, ct->world_age); if (jl_bkind_is_some_guard(jl_binding_kind(ownerbpart))) { jl_printf(JL_STDERR, @@ -965,24 +965,24 @@ static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, JL_UNLOCK(&world_counter_lock); } -JL_DLLEXPORT void jl_module_import(jl_module_t *to, jl_module_t *from, jl_sym_t *s) +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) { - module_import_(to, from, s, s, 1); + module_import_(ct, to, from, s, s, 1); } -JL_DLLEXPORT void jl_module_import_as(jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) +JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) { - module_import_(to, from, asname, s, 1); + module_import_(ct, to, from, asname, s, 1); } -JL_DLLEXPORT void jl_module_use(jl_module_t *to, jl_module_t *from, jl_sym_t *s) +JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) { - module_import_(to, from, s, s, 0); + module_import_(ct, to, from, s, s, 0); } -JL_DLLEXPORT void jl_module_use_as(jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) +JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) { - module_import_(to, from, asname, s, 0); + module_import_(ct, to, from, asname, s, 0); } void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) diff --git a/src/toplevel.c b/src/toplevel.c index dd9adb9bb480f0..575f0ac2adbe67 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -488,20 +488,18 @@ static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int } extern size_t jl_require_world; -static jl_module_t *call_require(jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_ROOTED +static jl_module_t *call_require(jl_task_t *ct, jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_ROOTED { JL_TIMING(LOAD_IMAGE, LOAD_Require); jl_timing_printf(JL_TIMING_DEFAULT_BLOCK, "%s", jl_symbol_name(var)); int build_mode = jl_options.incremental && jl_generating_output(); jl_module_t *m = NULL; - jl_task_t *ct = jl_current_task; static jl_value_t *require_func = NULL; if (require_func == NULL && jl_base_module != NULL) { require_func = jl_get_global(jl_base_module, jl_symbol("require")); } if (require_func != NULL) { - size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); if (build_mode && jl_require_world < ct->world_age) ct->world_age = jl_require_world; @@ -510,18 +508,19 @@ static jl_module_t *call_require(jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_RO reqargs[1] = (jl_value_t*)mod; reqargs[2] = (jl_value_t*)var; m = (jl_module_t*)jl_apply(reqargs, 3); - ct->world_age = last_age; } if (m == NULL || !jl_is_module(m)) { jl_errorf("failed to load module %s", jl_symbol_name(var)); } + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); return m; } // either: // - sets *name and returns the module to import *name from // - sets *name to NULL and returns a module to import -static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PROPAGATES_ROOT, +// also updates world_age +static jl_module_t *eval_import_path(jl_task_t *ct, jl_module_t *where, jl_module_t *from JL_PROPAGATES_ROOT, jl_array_t *args, jl_sym_t **name, const char *keyword) JL_GLOBALLY_ROOTED { if (jl_array_nrows(args) == 0) @@ -546,7 +545,7 @@ static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PR m = jl_base_module; } else { - m = call_require(where, var); + m = call_require(ct, where, var); } if (i == jl_array_nrows(args)) return m; @@ -566,6 +565,8 @@ static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PR } } + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + while (1) { var = (jl_sym_t*)jl_array_ptr_ref(args, i); if (!jl_is_symbol(var)) @@ -650,17 +651,17 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_instance_for_thunk(jl_code_info_t * return mi; } -static void import_module(jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) +static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) { assert(m); jl_sym_t *name = asname ? asname : import->name; // TODO: this is a bit race-y with what error message we might print jl_binding_t *b = jl_get_module_binding(m, name, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); if (kind != BINDING_KIND_GUARD && kind != BINDING_KIND_FAILED && kind != BINDING_KIND_DECLARED && kind != BINDING_KIND_IMPLICIT) { // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + jl_walk_binding_inplace(&b, &bpart, ct->world_age); if (jl_binding_kind(bpart) == BINDING_KIND_CONST || jl_binding_kind(bpart) == BINDING_KIND_BACKDATED_CONST || jl_binding_kind(bpart) == BINDING_KIND_CONST_IMPORT) { // Already declared (e.g. on another thread) or imported. if (bpart->restriction == (jl_value_t*)import) @@ -673,7 +674,7 @@ static void import_module(jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym } // in `import A.B: x, y, ...`, evaluate the `A.B` part if it exists -static jl_module_t *eval_import_from(jl_module_t *m JL_PROPAGATES_ROOT, jl_expr_t *ex, const char *keyword) +static jl_module_t *eval_import_from(jl_task_t *ct, jl_module_t *m JL_PROPAGATES_ROOT, jl_expr_t *ex, const char *keyword) { if (jl_expr_nargs(ex) == 1 && jl_is_expr(jl_exprarg(ex, 0))) { jl_expr_t *fr = (jl_expr_t*)jl_exprarg(ex, 0); @@ -682,7 +683,7 @@ static jl_module_t *eval_import_from(jl_module_t *m JL_PROPAGATES_ROOT, jl_expr_ jl_expr_t *path = (jl_expr_t*)jl_exprarg(fr, 0); if (((jl_expr_t*)path)->head == jl_dot_sym) { jl_sym_t *name = NULL; - jl_module_t *from = eval_import_path(m, NULL, path->args, &name, keyword); + jl_module_t *from = eval_import_path(ct, m, NULL, path->args, &name, keyword); if (name != NULL) { from = (jl_module_t*)jl_eval_global_var(from, name); if (!from || !jl_is_module(from)) @@ -828,8 +829,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val } else if (head == jl_using_sym) { jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(m, ex, "using"); - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_module_t *from = eval_import_from(ct, m, ex, "using"); size_t i = 0; if (from) { i = 1; @@ -839,10 +839,10 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val jl_value_t *a = jl_exprarg(ex, i); if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { name = NULL; - jl_module_t *import = eval_import_path(m, from, ((jl_expr_t*)a)->args, &name, "using"); + jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "using"); if (from) { // `using A: B` and `using A: B.c` syntax - jl_module_use(m, import, name); + jl_module_use(ct, m, import, name); } else { jl_module_t *u = import; @@ -857,7 +857,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val if (m == jl_main_module && name == NULL) { // TODO: for now, `using A` in Main also creates an explicit binding for `A` // This will possibly be extended to all modules. - import_module(m, u, NULL); + import_module(ct, m, u, NULL); } } continue; @@ -868,12 +868,11 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val if (jl_is_symbol(asname)) { jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); name = NULL; - jl_module_t *import = eval_import_path(m, from, ((jl_expr_t*)path)->args, &name, "using"); - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "using"); assert(name); check_macro_rename(name, asname, "using"); // `using A: B as C` syntax - jl_module_use_as(m, import, name, asname); + jl_module_use_as(ct, m, import, name, asname); continue; } } @@ -886,8 +885,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val } else if (head == jl_import_sym) { jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(m, ex, "import"); - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_module_t *from = eval_import_from(ct, m, ex, "import"); size_t i = 0; if (from) { i = 1; @@ -897,14 +895,14 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val jl_value_t *a = jl_exprarg(ex, i); if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { name = NULL; - jl_module_t *import = eval_import_path(m, from, ((jl_expr_t*)a)->args, &name, "import"); + jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "import"); if (name == NULL) { // `import A` syntax - import_module(m, import, NULL); + import_module(ct, m, import, NULL); } else { // `import A.B` or `import A: B` syntax - jl_module_import(m, import, name); + jl_module_import(ct, m, import, name); } continue; } @@ -914,15 +912,15 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val if (jl_is_symbol(asname)) { jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); name = NULL; - jl_module_t *import = eval_import_path(m, from, ((jl_expr_t*)path)->args, &name, "import"); + jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "import"); if (name == NULL) { // `import A as B` syntax - import_module(m, import, asname); + import_module(ct, m, import, asname); } else { check_macro_rename(name, asname, "import"); // `import A.B as C` syntax - jl_module_import_as(m, import, name, asname); + jl_module_import_as(ct, m, import, name, asname); } continue; } diff --git a/test/worlds.jl b/test/worlds.jl index f4558327c744b1..686708c5efd27f 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -524,3 +524,13 @@ module AmbigWorldTest convert(Core.Binding, GlobalRef(M2, :x)).partitions.min_world ) end + +module X57316; module Y57316; end; end +module A57316; using ..X57316.Y57316, .Y57316.Y57316; end +module B57316; import ..X57316.Y57316, .Y57316.Y57316; end +module C57316; import ..X57316.Y57316 as Z, .Z.Y57316 as W; end +@test X57316.Y57316 === A57316.Y57316 === B57316.Y57316 === C57316.Z === C57316.W +@test !isdefined(A57316, :X57316) +@test !isdefined(B57316, :X57316) +@test !isdefined(C57316, :X57316) +@test !isdefined(C57316, :Y57316) From 0fb5fa0276675c54b7dcd3b69e6ec95264f0b2a1 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 19 Feb 2025 19:27:27 -0500 Subject: [PATCH 06/25] staticdata: Don't use `newm` pointer after it has been invalidated (#57471) The buffer may end up reallocated by the additional writes performed to it in this function. --- src/staticdata.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index bc69b6a683821f..6cb157ad299fc8 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1344,6 +1344,10 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings_backedges))); arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings_backedges, s->link_ids_relocs)); + // After reload, everything that has happened in this process happened semantically at + // (for .incremental) or before jl_require_world, so reset this flag. + jl_atomic_store_relaxed(&newm->export_set_changed_since_require_world, 0); + // write out the usings list memset(&newm->usings._space, 0, sizeof(newm->usings._space)); if (m->usings.items == &m->usings._space[0]) { @@ -1372,6 +1376,7 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t newm->usings.items = (void**)tot; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items))); arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); + newm = NULL; // `write_*(s->s)` below may invalidate `newm`, so defensively set it to NULL size_t i; for (i = 0; i < module_usings_length(m); i++) { struct _jl_module_using *data = module_usings_getidx(m, i); @@ -1395,10 +1400,6 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t } } assert(ios_pos(s->s) - reloc_offset == tot); - - // After reload, everything that has happened in this process happened semantically at - // (for .incremental) or before jl_require_world, so reset this flag. - jl_atomic_store_relaxed(&newm->export_set_changed_since_require_world, 0); } static void record_memoryref(jl_serializer_state *s, size_t reloc_offset, jl_genericmemoryref_t ref) { From 378a42508d4ba40770adcf3833aef26056efde98 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Fri, 21 Feb 2025 01:53:47 +1100 Subject: [PATCH 07/25] Refactoring write barrier for copying generic memory, but moving it from `genericmemory.c` (#57252) This PR moves stock-specific write barriers from `jl_genericmemory_copyto` into `julia.h`. Note that this PR can be merged independently, but is heavily connected with https://github.com/JuliaLang/julia/pull/57237. Ideally they should be combined, but this PR actually introduces new functions to the GC interface, and that is the main reason I'd like them to be reviewed separately. The code for either PR will have to be adapted depending on which one gets merged first. --- src/gc-interface.h | 15 ++++++ src/genericmemory.c | 43 +--------------- src/julia.h | 122 ++++++++++++++++++++++++++++++++------------ 3 files changed, 106 insertions(+), 74 deletions(-) diff --git a/src/gc-interface.h b/src/gc-interface.h index 618077b1278031..0ac6057ee93157 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -8,6 +8,7 @@ #define JL_GC_INTERFACE_H #include "dtypes.h" +#include "julia_atomics.h" #ifdef __cplusplus extern "C" { @@ -17,6 +18,7 @@ struct _jl_tls_states_t; struct _jl_value_t; struct _jl_weakref_t; struct _jl_datatype_t; +struct _jl_genericmemory_t; // ========================================================================= // // GC Metrics @@ -253,6 +255,19 @@ STATIC_INLINE void jl_gc_wb_knownold(const void *parent, const void *ptr) JL_NOT STATIC_INLINE void jl_gc_multi_wb(const void *parent, const struct _jl_value_t *ptr) JL_NOTSAFEPOINT; +// Write-barrier function that must be used after copying fields of elements of genericmemory objects +// into another. It should be semantically equivalent to triggering multiple write barriers – one +// per field of the object being copied, but may be special-cased for performance reasons. +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const struct _jl_value_t *owner, struct _jl_genericmemory_t *src, char* src_p, + size_t n, struct _jl_datatype_t *dt) JL_NOTSAFEPOINT; + +// Similar to jl_gc_wb_genericmemory_copy but must be used when copying *boxed* elements of a genericmemory +// object. Note that this barrier also performs the copying unlike jl_gc_wb_genericmemory_copy_ptr. +// The parameters src_p, dest_p and n will be modified and will contain information about +// the *uncopied* data after performing this barrier, and will be copied using memmove_refs. +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const struct _jl_value_t *owner, _Atomic(void*) * dest_p, + struct _jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT; #ifdef __cplusplus } #endif diff --git a/src/genericmemory.c b/src/genericmemory.c index e435ec3b63c9f1..b455a2fb362741 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -235,36 +235,7 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda _Atomic(void*) * dest_p = (_Atomic(void*)*)destdata; _Atomic(void*) * src_p = (_Atomic(void*)*)srcdata; jl_value_t *owner = jl_genericmemory_owner(dest); - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - ssize_t done = 0; - if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { - if (dest_p < src_p || dest_p > src_p + n) { - for (; done < n; done++) { // copy forwards - void *val = jl_atomic_load_relaxed(src_p + done); - jl_atomic_store_release(dest_p + done, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - break; - } - } - src_p += done; - dest_p += done; - } else { - for (; done < n; done++) { // copy backwards - void *val = jl_atomic_load_relaxed(src_p + n - done - 1); - jl_atomic_store_release(dest_p + n - done - 1, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - break; - } - } - } - n -= done; - } - } + jl_gc_wb_genericmemory_copy_boxed(owner, dest_p, src, src_p, &n); return memmove_refs(dest_p, src_p, n); } size_t elsz = layout->size; @@ -280,17 +251,7 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda if (layout->first_ptr != -1) { memmove_refs((_Atomic(void*)*)destdata, (_Atomic(void*)*)srcdata, n * elsz / sizeof(void*)); jl_value_t *owner = jl_genericmemory_owner(dest); - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { - dt = (jl_datatype_t*)jl_tparam1(dt); - for (size_t done = 0; done < n; done++) { // copy forwards - char* s = (char*)src_p+done*elsz; - if (*((jl_value_t**)s+layout->first_ptr) != NULL) - jl_gc_queue_multiroot(owner, s, dt); - } - } - } + jl_gc_wb_genericmemory_copy_ptr(owner, src, src_p, n, dt); } else { memmove(destdata, srcdata, n * elsz); diff --git a/src/julia.h b/src/julia.h index 3206b4f1f033be..49c43c40c7b44d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -162,7 +162,7 @@ typedef struct { // jl_value_t *data[]; } jl_svec_t; -JL_EXTENSION typedef struct { +JL_EXTENSION typedef struct _jl_genericmemory_t { JL_DATA_TYPE size_t length; void *ptr; @@ -1182,38 +1182,6 @@ JL_DLLEXPORT void jl_free_stack(void *stkbuf, size_t bufsz); // thread-local allocator of the current thread. JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value); -// GC write barriers - -STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT -{ - // parent and ptr isa jl_value_t* - if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset - (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young - jl_gc_queue_root((jl_value_t*)parent); -} - -STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* -{ - // if ptr is old - if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { - jl_gc_queue_root((jl_value_t*)ptr); - } -} - -STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT -{ - // 3 == GC_OLD_MARKED - // ptr is an immutable object - if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) - return; // parent is young or in remset - if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) - return; // ptr is old and not in remset (thus it does not point to young) - jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); - const jl_datatype_layout_t *ly = dt->layout; - if (ly->npointers) - jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); -} - JL_DLLEXPORT void jl_gc_safepoint(void); JL_DLLEXPORT int jl_safepoint_suspend_thread(int tid, int waitstate); JL_DLLEXPORT void jl_safepoint_suspend_all_threads(struct _jl_task_t *ct); @@ -1332,6 +1300,94 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set( } #endif +// GC write barriers + +STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT +{ + // parent and ptr isa jl_value_t* + if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset + (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young + jl_gc_queue_root((jl_value_t*)parent); +} + +STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* +{ + // if ptr is old + if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { + jl_gc_queue_root((jl_value_t*)ptr); + } +} + +STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT +{ + // 3 == GC_OLD_MARKED + // ptr is an immutable object + if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) + return; // parent is young or in remset + if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) + return; // ptr is old and not in remset (thus it does not point to young) + jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); + const jl_datatype_layout_t *ly = dt->layout; + if (ly->npointers) + jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); +} + +STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, + jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t done = 0; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + if (dest_p < src_p || dest_p > src_p + (*n)) { + for (; done < (*n); done++) { // copy forwards + void *val = jl_atomic_load_relaxed(src_p + done); + jl_atomic_store_release(dest_p + done, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + src_p += done; + dest_p += done; + } + else { + for (; done < (*n); done++) { // copy backwards + void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); + jl_atomic_store_release(dest_p + (*n) - done - 1, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + } + (*n) -= done; + } + } +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, + size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t elsz = dt->layout->size; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + dt = (jl_datatype_t*)jl_tparam1(dt); + for (size_t done = 0; done < n; done++) { // copy forwards + char* s = (char*)src_p+done*elsz; + if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) + jl_gc_queue_multiroot(owner, s, dt); + } + } + } +} + STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT { jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; From 7e6971024b2df523b15d8a50fba3abe2dbd2977c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 20 Feb 2025 10:22:47 -0500 Subject: [PATCH 08/25] binaryplatforms.jl: Remove unnecessary backslash in regex (#57468) This regex and two very similar ones a few lines below use different "quoting styles", which confused me when reading, suspecting an error. There is no error. This PR changes `\.` to `.` inside a regex character class. Both have the same meaning. Avoiding the backslash is the simpler way. --- base/binaryplatforms.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 598b618f0d1ed6..0a68ce56e9a6b6 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -751,7 +751,7 @@ function Base.parse(::Type{Platform}, triplet::String; validate_strict::Bool = f end os_version = nothing if os == "macos" - os_version = extract_os_version("macos", r".*darwin([\d\.]+)"sa) + os_version = extract_os_version("macos", r".*darwin([\d.]+)"sa) end if os == "freebsd" os_version = extract_os_version("freebsd", r".*freebsd([\d.]+)"sa) From 940928eb761ca0a92b14aa45ade22558f4c915c2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 20 Feb 2025 16:37:21 -0500 Subject: [PATCH 09/25] codegen: cleanup gcstack call frames somewhat earlier (#57410) Slightly reduces the amount of work these optimization passes need to do later, in most typical cases, and avoids putting an unknown call at the top of all of our functions, which can inhibit some optimizations of otherwise trivial functions. Fixes #57400 --- src/ccall.cpp | 4 +- src/codegen.cpp | 110 ++++++++++++++++++--------------- src/llvm-final-gc-lowering.cpp | 27 +++++--- src/llvm-gc-interface-passes.h | 7 ++- src/llvm-late-gc-lowering.cpp | 18 +++--- src/llvm-pass-helpers.cpp | 35 +++-------- src/llvm-pass-helpers.h | 8 +-- src/llvm-ptls.cpp | 6 +- 8 files changed, 110 insertions(+), 105 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index c35979eb85b1dc..f4cb5cf7bc05bf 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -2162,12 +2162,12 @@ jl_cgval_t function_sig_t::emit_a_ccall( } } - // Potentially we could drop `jl_roots(gc_uses)` in the presence of `gc-transition(gc_uses)` + // Potentially we could add gc_uses to `gc-transition`, instead of emitting them separately as jl_roots SmallVector bundles; if (!gc_uses.empty()) bundles.push_back(OperandBundleDef("jl_roots", gc_uses)); if (gc_safe) - bundles.push_back(OperandBundleDef("gc-transition", ArrayRef {})); + bundles.push_back(OperandBundleDef("gc-transition", get_current_ptls(ctx))); // the actual call CallInst *ret = ctx.builder.CreateCall(functype, llvmf, argvals, diff --git a/src/codegen.cpp b/src/codegen.cpp index ac74e1443685a3..a504bb189bdfd4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7002,8 +7002,9 @@ static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=fal // allocate a placeholder gc instruction // this will require the runtime, but it gets deleted later if unused ctx.topalloca = ctx.builder.CreateCall(prepare_call(or_new ? jladoptthread_func : jlpgcstack_func)); - ctx.pgcstack = ctx.topalloca; - ctx.pgcstack->setName("pgcstack"); + ctx.topalloca->setName("pgcstack"); + if (ctx.pgcstack == nullptr) + ctx.pgcstack = ctx.topalloca; } static Value *get_current_task(jl_codectx_t &ctx) @@ -7150,7 +7151,6 @@ static void emit_specsig_to_specsig( ctx.builder.SetInsertPoint(b0); DebugLoc noDbg; ctx.builder.SetCurrentDebugLocation(noDbg); - allocate_gc_frame(ctx, b0); Function::arg_iterator AI = gf_thunk->arg_begin(); SmallVector myargs(nargs); if (cc == jl_returninfo_t::SRet || cc == jl_returninfo_t::Union) @@ -7158,8 +7158,10 @@ static void emit_specsig_to_specsig( if (return_roots) ++AI; if (JL_FEAT_TEST(ctx,gcstack_arg)) { + ctx.pgcstack = AI; ++AI; // gcstack_arg } + allocate_gc_frame(ctx, b0); for (size_t i = 0; i < nargs; i++) { if (i == 0 && is_for_opaque_closure) { // `jt` would be wrong here (it is the captures type), so is not used used for @@ -7266,6 +7268,10 @@ static void emit_specsig_to_specsig( break; } } + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } } void emit_specsig_to_fptr1( @@ -8122,6 +8128,10 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_va CreateTrap(ctx.builder, false); else ctx.builder.CreateRet(boxed(ctx, retval)); + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } } static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, @@ -8778,7 +8788,53 @@ static jl_llvm_functions_t ctx.spvals_ptr = &*AI++; } } - // step 6. set up GC frame + // step 6. set up GC frame and special arguments + Function::arg_iterator AI = f->arg_begin(); + SmallVector attrs(f->arg_size()); // function declaration attributes + + if (has_sret) { + Argument *Arg = &*AI; + ++AI; + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + if (returninfo.cc == jl_returninfo_t::Union) { + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + param.addDereferenceableAttr(returninfo.union_bytes); + param.addAlignmentAttr(returninfo.union_align); + } + else { + const DataLayout &DL = jl_Module->getDataLayout(); + Type *RT = Arg->getParamStructRetType(); + TypeSize sz = DL.getTypeAllocSize(RT); + Align al = DL.getPrefTypeAlign(RT); + if (al > MAX_ALIGN) + al = Align(MAX_ALIGN); + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + param.addDereferenceableAttr(sz); + param.addAlignmentAttr(al); + } + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes + } + if (returninfo.return_roots) { + Argument *Arg = &*AI; + ++AI; + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + size_t size = returninfo.return_roots * sizeof(jl_value_t*); + param.addDereferenceableAttr(size); + param.addAlignmentAttr(Align(sizeof(jl_value_t*))); + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes + } + if (specsig && JL_FEAT_TEST(ctx, gcstack_arg)) { + Argument *Arg = &*AI; + ctx.pgcstack = Arg; + ++AI; + AttrBuilder param(ctx.builder.getContext()); + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); + } + allocate_gc_frame(ctx, b0); Value *last_age = NULL; Value *world_age_field = NULL; @@ -8921,9 +8977,6 @@ static jl_llvm_functions_t } // step 8. move args into local variables - Function::arg_iterator AI = f->arg_begin(); - SmallVector attrs(f->arg_size()); // function declaration attributes - auto get_specsig_arg = [&](jl_value_t *argType, Type *llvmArgType, bool isboxed) { if (type_is_ghost(llvmArgType)) { // this argument is not actually passed return ghostValue(ctx, argType); @@ -8956,47 +9009,6 @@ static jl_llvm_functions_t return theArg; }; - if (has_sret) { - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); - if (returninfo.cc == jl_returninfo_t::Union) { - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - param.addDereferenceableAttr(returninfo.union_bytes); - param.addAlignmentAttr(returninfo.union_align); - } - else { - const DataLayout &DL = jl_Module->getDataLayout(); - Type *RT = Arg->getParamStructRetType(); - TypeSize sz = DL.getTypeAllocSize(RT); - Align al = DL.getPrefTypeAlign(RT); - if (al > MAX_ALIGN) - al = Align(MAX_ALIGN); - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - param.addDereferenceableAttr(sz); - param.addAlignmentAttr(al); - } - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes - } - if (returninfo.return_roots) { - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - size_t size = returninfo.return_roots * sizeof(jl_value_t*); - param.addDereferenceableAttr(size); - param.addAlignmentAttr(Align(sizeof(jl_value_t*))); - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes - } - if (specsig && JL_FEAT_TEST(ctx, gcstack_arg)){ - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext()); - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); - } for (i = 0; i < nreq && i < vinfoslen; i++) { jl_sym_t *s = slot_symbol(ctx, i); jl_varinfo_t &vi = ctx.slots[i]; @@ -9964,7 +9976,7 @@ static jl_llvm_functions_t } } - if (ctx.topalloca->use_empty()) { + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { ctx.topalloca->eraseFromParent(); ctx.topalloca = nullptr; } diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 50bdd2251ab42e..5525d079d3f91f 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -161,16 +161,23 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F) target->replaceAllUsesWith(newI); target->eraseFromParent(); } -bool FinalLowerGC::shouldRunFinalGC(Function &F) + +static bool hasUse(const JuliaPassContext &ctx, const jl_intrinsics::IntrinsicDescription &v) +{ + auto Intr = ctx.getOrNull(v); + return Intr && !Intr->use_empty(); +} + +bool FinalLowerGC::shouldRunFinalGC() { bool should_run = 0; - should_run |= getOrNull(jl_intrinsics ::newGCFrame) != nullptr; - should_run |= getOrNull(jl_intrinsics ::getGCFrameSlot) != nullptr; - should_run |= getOrNull(jl_intrinsics ::pushGCFrame) != nullptr; - should_run |= getOrNull(jl_intrinsics ::popGCFrame) != nullptr; - should_run |= getOrNull(jl_intrinsics ::GCAllocBytes) != nullptr; - should_run |= getOrNull(jl_intrinsics ::queueGCRoot) != nullptr; - should_run |= getOrNull(jl_intrinsics ::safepoint) != nullptr; + should_run |= hasUse(*this, jl_intrinsics::newGCFrame); + should_run |= hasUse(*this, jl_intrinsics::getGCFrameSlot); + should_run |= hasUse(*this, jl_intrinsics::pushGCFrame); + should_run |= hasUse(*this, jl_intrinsics::popGCFrame); + should_run |= hasUse(*this, jl_intrinsics::GCAllocBytes); + should_run |= hasUse(*this, jl_intrinsics::queueGCRoot); + should_run |= hasUse(*this, jl_intrinsics::safepoint); return should_run; } @@ -178,7 +185,7 @@ bool FinalLowerGC::runOnFunction(Function &F) { initAll(*F.getParent()); pgcstack = getPGCstack(F); - if (!shouldRunFinalGC(F)) + if (!pgcstack || !shouldRunFinalGC()) goto verify_skip; LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n"); @@ -232,7 +239,7 @@ bool FinalLowerGC::runOnFunction(Function &F) auto IS_INTRINSIC = [&](auto intrinsic) { auto intrinsic2 = getOrNull(intrinsic); if (intrinsic2 == callee) { - errs() << "Final-GC-lowering didn't eliminate all intrinsics'" << F.getName() << "', dumping entire module!\n\n"; + errs() << "Final-GC-lowering didn't eliminate all intrinsics from '" << F.getName() << "', dumping entire module!\n\n"; errs() << *F.getParent() << "\n"; abort(); } diff --git a/src/llvm-gc-interface-passes.h b/src/llvm-gc-interface-passes.h index b899cd9494673f..028337be12e9f4 100644 --- a/src/llvm-gc-interface-passes.h +++ b/src/llvm-gc-interface-passes.h @@ -328,7 +328,7 @@ struct LateLowerGCFrame: private JuliaPassContext { bool runOnFunction(Function &F, bool *CFGModified = nullptr); private: - CallInst *pgcstack; + Value *pgcstack; Function *smallAllocFunc; void MaybeNoteDef(State &S, BBState &BBS, Value *Def, const ArrayRef &SafepointsSoFar, @@ -388,7 +388,7 @@ struct FinalLowerGC: private JuliaPassContext { Function *smallAllocFunc; Function *bigAllocFunc; Function *allocTypedFunc; - Instruction *pgcstack; + Value *pgcstack; Type *T_size; // Lowers a `julia.new_gc_frame` intrinsic. @@ -411,8 +411,9 @@ struct FinalLowerGC: private JuliaPassContext { // Lowers a `julia.safepoint` intrinsic. void lowerSafepoint(CallInst *target, Function &F); + // Check if the pass should be run - bool shouldRunFinalGC(Function &F); + bool shouldRunFinalGC(); }; #endif // LLVM_GC_PASSES_H diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 32ff9e04cf6d0a..b8e732f06190ae 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -2219,20 +2219,21 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { SmallVector bundles; CI->getOperandBundlesAsDefs(bundles); bool gc_transition = false; + Value *ptls; for (auto &bundle: bundles) - if (bundle.getTag() == "gc-transition") + if (bundle.getTag() == "gc-transition") { gc_transition = true; + ptls = bundle.inputs()[0]; + } // In theory LLVM wants us to lower this using RewriteStatepointsForGC if (gc_transition) { // Insert the operations to switch to gc_safe if necessary. IRBuilder<> builder(CI); - Value *pgcstack = getOrAddPGCstack(F); - assert(pgcstack); + assert(ptls); // We dont use emit_state_set here because safepoints are unconditional for any code that reaches this // We are basically guaranteed to go from gc_unsafe to gc_safe and back, and both transitions need a safepoint // We also can't add any BBs here, so just avoiding the branches is good - Value *ptls = get_current_ptls_from_task(builder, get_current_task_from_pgcstack(builder, pgcstack), tbaa_gcframe); unsigned offset = offsetof(jl_tls_states_t, gc_state); Value *gc_state = builder.CreateConstInBoundsGEP1_32(Type::getInt8Ty(builder.getContext()), ptls, offset, "gc_state"); LoadInst *last_gc_state = builder.CreateAlignedLoad(Type::getInt8Ty(builder.getContext()), gc_state, Align(sizeof(void*))); @@ -2249,7 +2250,7 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { ++it; continue; } else { - // remove operand bundle + // remove all operand bundles CallInst *NewCall = CallInst::Create(CI, None, CI); NewCall->takeName(CI); NewCall->copyMetadata(*CI); @@ -2395,7 +2396,10 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss auto pushGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::pushGCFrame), {gcframe, ConstantInt::get(T_int32, 0)}); - pushGcframe->insertAfter(pgcstack); + if (isa(pgcstack)) + pushGcframe->insertAfter(gcframe); + else + pushGcframe->insertAfter(cast(pgcstack)); // we don't run memsetopt after this, so run a basic approximation of it // that removes any redundant memset calls in the prologue since getGCFrameSlot already includes the null store @@ -2513,8 +2517,6 @@ bool LateLowerGCFrame::runOnFunction(Function &F, bool *CFGModified) { initAll(*F.getParent()); smallAllocFunc = getOrDeclare(jl_well_known::GCSmallAlloc); LLVM_DEBUG(dbgs() << "GC ROOT PLACEMENT: Processing function " << F.getName() << "\n"); - if (!pgcstack_getter && !adoptthread_func) - return CleanupIR(F, nullptr, CFGModified); pgcstack = getPGCstack(F); if (!pgcstack) diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 9d415d923ecb65..38716cba521a1e 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -72,25 +72,9 @@ void JuliaPassContext::initAll(Module &M) T_prjlvalue = JuliaType::get_prjlvalue_ty(ctx); } -llvm::CallInst *JuliaPassContext::getPGCstack(llvm::Function &F) const +llvm::Value *JuliaPassContext::getPGCstack(llvm::Function &F) const { - if (!pgcstack_getter && !adoptthread_func) - return nullptr; - for (auto &I : F.getEntryBlock()) { - if (CallInst *callInst = dyn_cast(&I)) { - Value *callee = callInst->getCalledOperand(); - if ((pgcstack_getter && callee == pgcstack_getter) || - (adoptthread_func && callee == adoptthread_func)) { - return callInst; - } - } - } - return nullptr; -} - -llvm::CallInst *JuliaPassContext::getOrAddPGCstack(llvm::Function &F) -{ - if (pgcstack_getter || adoptthread_func) + if (pgcstack_getter || adoptthread_func) { for (auto &I : F.getEntryBlock()) { if (CallInst *callInst = dyn_cast(&I)) { Value *callee = callInst->getCalledOperand(); @@ -100,13 +84,14 @@ llvm::CallInst *JuliaPassContext::getOrAddPGCstack(llvm::Function &F) } } } - IRBuilder<> builder(&F.getEntryBlock().front()); - if (pgcstack_getter) - return builder.CreateCall(pgcstack_getter); - auto FT = FunctionType::get(PointerType::get(F.getContext(), 0), false); - auto F2 = Function::Create(FT, Function::ExternalLinkage, "julia.get_pgcstack", F.getParent()); - pgcstack_getter = F2; - return builder.CreateCall( F2); + } + if (F.getCallingConv() == CallingConv::Swift) { + for (auto &arg : F.args()) { + if (arg.hasSwiftSelfAttr()) + return &arg; + } + } + return nullptr; } llvm::Function *JuliaPassContext::getOrNull( diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index ac08cda2d61e0b..d79470818c2878 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -85,12 +85,10 @@ struct JuliaPassContext { // Gets a call to the `julia.get_pgcstack' intrinsic in the entry // point of the given function, if there exists such a call. + // Otherwise, gets a swiftself argument, if there exists such an argument. // Otherwise, `nullptr` is returned. - llvm::CallInst *getPGCstack(llvm::Function &F) const; - // Gets a call to the `julia.get_pgcstack' intrinsic in the entry - // point of the given function, if there exists such a call. - // Otherwise, creates a new call to the intrinsic - llvm::CallInst *getOrAddPGCstack(llvm::Function &F); + llvm::Value *getPGCstack(llvm::Function &F) const; + // Gets the intrinsic or well-known function that conforms to // the given description if it exists in the module. If not, // `nullptr` is returned. diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 15f5a5574a6d31..238032e68d3045 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -322,13 +322,13 @@ bool LowerPTLS::run(bool *CFGModified) need_init = false; } - for (auto it = pgcstack_getter->user_begin(); it != pgcstack_getter->user_end();) { + for (auto it = pgcstack_getter->user_begin(); it != pgcstack_getter->user_end(); ) { auto call = cast(*it); ++it; auto f = call->getCaller(); Value *pgcstack = NULL; - for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end();++arg) { - if (arg->hasSwiftSelfAttr()){ + for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end(); ++arg) { + if (arg->hasSwiftSelfAttr()) { pgcstack = &*arg; break; } From 414aca21e936c3605038eb35a8f58f11067317b4 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:25:38 -0800 Subject: [PATCH 10/25] lowering: Don't mutate lambda in `linearize` (#57416) Fixes #56904. The associated PR (#55876) compiles a finally block, then compiles a renumbered version of it. This works if `compile` doesn't mutate its input, but in reality, lambda bodies were being `set!` when linearized. The "invalid syntax" error was a result of attempting to linearize a lambda twice. --- src/julia-syntax.scm | 9 ++++----- src/utils.scm | 13 +++++++++++++ test/syntax.jl | 4 ++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index b6dc7bb9c8a122..47cbc9dec1503a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -4322,11 +4322,10 @@ f(x) = yt(x) (define (linearize e) (cond ((or (not (pair? e)) (quoted? e)) e) ((eq? (car e) 'lambda) - (set-car! (cdddr e) (compile-body (cadddr e) (append (car (caddr e)) - (cadr (caddr e))) - e))) - (else (for-each linearize (cdr e)))) - e) + (list-set e 3 (compile-body (cadddr e) + (append (car (caddr e)) + (cadr (caddr e))) e))) + (else (cons (car e) (map linearize (cdr e)))))) (define (valid-ir-argument? e) (or (simple-atom? e) diff --git a/src/utils.scm b/src/utils.scm index 79e3a280b9886c..80fc44615a49aa 100644 --- a/src/utils.scm +++ b/src/utils.scm @@ -119,3 +119,16 @@ (cons (car lst) (filter (lambda (x) (not (pred x))) (cdr lst)))) (else (cons (car lst) (keep-first pred (cdr lst)))))) + +(define (take lst n) + (let loop ((lst lst) (n n) (out '())) + (if (= n 0) (reverse out) + (loop (cdr lst) (- n 1) (cons (car lst) out))))) + +(define (drop lst n) + (if (= n 0) lst + (drop (cdr lst) (- n 1)))) + +;; functional update at position i +(define (list-set lst i val) + (append (take lst i) (list val) (drop lst (+ i 1)))) diff --git a/test/syntax.jl b/test/syntax.jl index 2230d01991691f..5b604a81d8b1b4 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4102,3 +4102,7 @@ module Ambig57404 using .B end @test Ambig57404.S == 1 + +# Issue #56904 - lambda linearized twice +@test (let; try 3; finally try 1; f(() -> x); catch x; end; end; x = 7; end) === 7 +@test (let; try 3; finally try 4; finally try 1; f(() -> x); catch x; end; end; end; x = 7; end) === 7 From 3c02af98dcad3dcbbab772af9fe1494082aea8ca Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:36:47 -0800 Subject: [PATCH 11/25] lowering: Handle malformed `...` expressions (#57480) Fixes #51572. The expander had a looser definition of a vararg than the recursive logic destructing the vararg, so a bad expression like `(call f (... x y))` could cause a stack overflow via indestructible mutant semi-vararg. This change produces a saner error in this situation: ``` julia> macro foo(); Expr(:(...), 1, 2, 3); end; (@foo,) ERROR: syntax: wrong number of expressions following "..." around REPL[1]:1 Stacktrace: [1] top-level scope @ REPL[1]:1 ``` --------- Co-authored-by: Jameson Nash --- src/ast.scm | 2 +- src/julia-syntax.scm | 8 +++++--- test/syntax.jl | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ast.scm b/src/ast.scm index 5cc97014e373e6..a20f8f87f955c9 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -449,7 +449,7 @@ (check-dotop (cadr e)))))) e) -(define (vararg? x) (and (pair? x) (eq? (car x) '...))) +(define (vararg? x) (and (pair? x) (eq? (car x) '...) (length= x 2))) (define (vararg-type-expr? x) (or (eq? x 'Vararg) (and (length> x 1) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 47cbc9dec1503a..6beff02cb1a0b8 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2691,8 +2691,7 @@ (if (null? run) '() (list `(call (core tuple) ,.(reverse run)))) (let ((x (car a))) - (if (and (length= x 2) - (eq? (car x) '...)) + (if (vararg? x) (if (null? run) (list* (cadr x) (tuple-wrap (cdr a) '())) @@ -2814,7 +2813,10 @@ '.>>>= lower-update-op '|...| - (lambda (e) (error "\"...\" expression outside call")) + (lambda (e) + (if (not (length= e 2)) + (error "wrong number of expressions following \"...\"")) + (error "\"...\" expression outside call")) '$ (lambda (e) (error "\"$\" expression outside quote")) diff --git a/test/syntax.jl b/test/syntax.jl index 5b604a81d8b1b4..a9bb19c128d81e 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -345,6 +345,10 @@ end # issue #15828 @test Meta.lower(Main, Meta.parse("x...")) == Expr(:error, "\"...\" expression outside call") +# issue #57153 - malformed "..." expr +@test Meta.lower(@__MODULE__, :(identity($(Expr(:(...), 1, 2, 3))))) == + (Expr(:error, "wrong number of expressions following \"...\"")) + # issue #15830 @test Meta.lower(Main, Meta.parse("foo(y = (global x)) = y")) == Expr(:error, "misplaced \"global\" declaration") From fb0a283e62dc38d2084aa09cf0b968858592e975 Mon Sep 17 00:00:00 2001 From: Ajinkya Kokandakar Date: Fri, 21 Feb 2025 06:08:02 -0600 Subject: [PATCH 12/25] fixed the URLs for the dependencies in the build doc, fixes #57469 (#57482) julia markdown parser does not support reference style links which was causing the dependency urls to not be rendered correctly in the build docs html. See issue #57469. --- doc/src/devdocs/build/build.md | 102 +++++++++++---------------------- 1 file changed, 33 insertions(+), 69 deletions(-) diff --git a/doc/src/devdocs/build/build.md b/doc/src/devdocs/build/build.md index fb7aaf59daa449..cb7a5bcfbed4bf 100644 --- a/doc/src/devdocs/build/build.md +++ b/doc/src/devdocs/build/build.md @@ -161,20 +161,20 @@ Notes for various architectures: Building Julia requires that the following software be installed: -- **[GNU make]** — building dependencies. -- **[gcc & g++][gcc]** (>= 7.1) or **[Clang][clang]** (>= 5.0, >= 9.3 for Apple Clang) — compiling and linking C, C++. -- **[libatomic][gcc]** — provided by **[gcc]** and needed to support atomic operations. -- **[python]** (>=2.7) — needed to build LLVM. -- **[gfortran]** — compiling and linking Fortran libraries. -- **[perl]** — preprocessing of header files of libraries. -- **[wget]**, **[curl]**, or **[fetch]** (FreeBSD) — to automatically download external libraries. -- **[m4]** — needed to build GMP. -- **[awk]** — helper tool for Makefiles. -- **[patch]** — for modifying source code. -- **[cmake]** (>= 3.4.3) — needed to build `libgit2`. -- **[pkg-config]** — needed to build `libgit2` correctly, especially for proxy support. -- **[powershell]** (>= 3.0) — necessary only on Windows. -- **[which]** — needed for checking build dependencies. +- **[GNU make](https://www.gnu.org/software/make)** — building dependencies. +- **[gcc & g++](https://gcc.gnu.org)** (>= 7.1) or **[Clang][clang]** (>= 5.0, >= 9.3 for Apple Clang) — compiling and linking C, C++. +- **[libatomic](https://gcc.gnu.org)** — provided by **[gcc]** and needed to support atomic operations. +- **[python](https://www.python.org/)** (>=2.7) — needed to build LLVM. +- **[gfortran](https://gcc.gnu.org/fortran/)** — compiling and linking Fortran libraries. +- **[perl](https://www.perl.org)** — preprocessing of header files of libraries. +- **[wget](https://www.gnu.org/software/wget)**, **[curl](https://curl.se)**, or **[fetch](https://www.freebsd.org/cgi/man.cgi?fetch(1))** (FreeBSD) — to automatically download external libraries. +- **[m4](https://www.gnu.org/software/m4)** — needed to build GMP. +- **[awk](https://www.gnu.org/software/gawk)** — helper tool for Makefiles. +- **[patch](https://www.gnu.org/software/patch)** — for modifying source code. +- **[cmake](https://www.cmake.org)** (>= 3.4.3) — needed to build `libgit2`. +- **[pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)** — needed to build `libgit2` correctly, especially for proxy support. +- **[powershell](https://docs.microsoft.com/en-us/powershell/scripting/wmf/overview)** (>= 3.0) — necessary only on Windows. +- **[which](https://carlowood.github.io/which/)** — needed for checking build dependencies. On Debian-based distributions (e.g. Ubuntu), you can easily install them with `apt-get`: ``` @@ -187,61 +187,25 @@ repository) and then compiled from source the first time you run `make`. The specific version numbers of these libraries that Julia uses are listed in [`deps/$(libname).version`](https://github.com/JuliaLang/julia/blob/master/deps/): -- **[LLVM]** (15.0 + [patches](https://github.com/JuliaLang/llvm-project/tree/julia-release/15.x)) — compiler infrastructure (see [note below](#llvm)). -- **[FemtoLisp]** — packaged with Julia source, and used to implement the compiler front-end. -- **[libuv]** (custom fork) — portable, high-performance event-based I/O library. -- **[OpenLibm]** — portable libm library containing elementary math functions. -- **[DSFMT]** — fast Mersenne Twister pseudorandom number generator library. -- **[OpenBLAS]** — fast, open, and maintained [basic linear algebra subprograms (BLAS)] -- **[LAPACK]** — library of linear algebra routines for solving systems of simultaneous linear equations, least-squares solutions of linear systems of equations, eigenvalue problems, and singular value problems. -- **[MKL]** (optional) – OpenBLAS and LAPACK may be replaced by Intel's MKL library. -- **[SuiteSparse]** — library of linear algebra routines for sparse matrices. -- **[PCRE]** — Perl-compatible regular expressions library. -- **[GMP]** — GNU multiple precision arithmetic library, needed for `BigInt` support. -- **[MPFR]** — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. -- **[libgit2]** — Git linkable library, used by Julia's package manager. -- **[curl]** — libcurl provides download and proxy support. -- **[libssh2]** — library for SSH transport, used by libgit2 for packages with SSH remotes. -- **[OpenSSL]** — library used for cryptography and transport layer security, used by libgit2 and libssh2. -- **[utf8proc]** — a library for processing UTF-8 encoded Unicode strings. -- **[LLVM libunwind]** — LLVM's fork of [libunwind], a library that determines the call-chain of a program. -- **[ITTAPI]** — Intel's Instrumentation and Tracing Technology and Just-In-Time API. - -[GNU make]: https://www.gnu.org/software/make -[patch]: https://www.gnu.org/software/patch -[wget]: https://www.gnu.org/software/wget -[m4]: https://www.gnu.org/software/m4 -[awk]: https://www.gnu.org/software/gawk -[gcc]: https://gcc.gnu.org -[clang]: https://clang.llvm.org -[python]: https://www.python.org/ -[gfortran]: https://gcc.gnu.org/fortran/ -[curl]: https://curl.haxx.se -[fetch]: https://www.freebsd.org/cgi/man.cgi?fetch(1) -[perl]: https://www.perl.org -[cmake]: https://www.cmake.org -[OpenLibm]: https://github.com/JuliaLang/openlibm -[DSFMT]: https://github.com/MersenneTwister-Lab/dSFMT -[OpenBLAS]: https://github.com/xianyi/OpenBLAS -[LAPACK]: https://www.netlib.org/lapack -[MKL]: https://software.intel.com/en-us/articles/intel-mkl -[SuiteSparse]: https://people.engr.tamu.edu/davis/suitesparse.html -[PCRE]: https://www.pcre.org -[LLVM]: https://www.llvm.org -[LLVM libunwind]: https://github.com/llvm/llvm-project/tree/main/libunwind -[FemtoLisp]: https://github.com/JeffBezanson/femtolisp -[GMP]: https://gmplib.org -[MPFR]: https://www.mpfr.org -[libuv]: https://github.com/JuliaLang/libuv -[libgit2]: https://libgit2.org/ -[utf8proc]: https://julialang.org/utf8proc/ -[libunwind]: https://www.nongnu.org/libunwind -[libssh2]: https://www.libssh2.org -[OpenSSL]: https://www.openssl.org/ -[pkg-config]: https://www.freedesktop.org/wiki/Software/pkg-config/ -[powershell]: https://docs.microsoft.com/en-us/powershell/scripting/wmf/overview -[which]: https://carlowood.github.io/which/ -[ITTAPI]: https://github.com/intel/ittapi +- **[LLVM](https://www.llvm.org)** (15.0 + [patches](https://github.com/JuliaLang/llvm-project/tree/julia-release/15.x)) — compiler infrastructure (see [note below](#llvm)). +- **[FemtoLisp](https://github.com/JeffBezanson/femtolisp)** — packaged with Julia source, and used to implement the compiler front-end. +- **[libuv](https://github.com/JuliaLang/libuv)** (custom fork) — portable, high-performance event-based I/O library. +- **[OpenLibm](https://github.com/JuliaLang/openlibm)** — portable libm library containing elementary math functions. +- **[DSFMT](https://github.com/MersenneTwister-Lab/dSFMT)** — fast Mersenne Twister pseudorandom number generator library. +- **[OpenBLAS](https://github.com/xianyi/OpenBLAS)** — fast, open, and maintained [basic linear algebra subprograms (BLAS)] +- **[LAPACK](https://www.netlib.org/lapack)** — library of linear algebra routines for solving systems of simultaneous linear equations, least-squares solutions of linear systems of equations, eigenvalue problems, and singular value problems. +- **[MKL](https://software.intel.com/en-us/articles/intel-mkl)** (optional) – OpenBLAS and LAPACK may be replaced by Intel's MKL library. +- **[SuiteSparse](https://people.engr.tamu.edu/davis/suitesparse.html)** — library of linear algebra routines for sparse matrices. +- **[PCRE](https://www.pcre.org)** — Perl-compatible regular expressions library. +- **[GMP](https://gmplib.org)** — GNU multiple precision arithmetic library, needed for `BigInt` support. +- **[MPFR](https://www.mpfr.org)** — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. +- **[libgit2](https://libgit2.org/)** — Git linkable library, used by Julia's package manager. +- **[curl](https://curl.se)** — libcurl provides download and proxy support. +- **[libssh2](https://www.libssh2.org)** — library for SSH transport, used by libgit2 for packages with SSH remotes. +- **[OpenSSL](https://www.openssl.org/)** — library used for cryptography and transport layer security, used by libgit2 and libssh2. +- **[utf8proc](https://julialang.org/utf8proc/)** — a library for processing UTF-8 encoded Unicode strings. +- **[LLVM libunwind](https://www.nongnu.org/libunwind)** — LLVM's fork of [libunwind], a library that determines the call-chain of a program. +- **[ITTAPI](https://github.com/intel/ittapi)** — Intel's Instrumentation and Tracing Technology and Just-In-Time API. ## Build dependencies From 58ce713c5a7b4c2825beb4f649da7547ca09b2bd Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Fri, 21 Feb 2025 09:40:15 -0300 Subject: [PATCH 13/25] Experiment with a slighly adjusted pipeline (#52850) Needs https://github.com/JuliaLang/julia/pull/57380 to merge first --- src/pipeline.cpp | 155 ++++++++++++++++++----------- test/boundscheck_exec.jl | 4 +- test/llvmpasses/image-codegen.jl | 3 +- test/llvmpasses/late-lower-gc.ll | 1 + test/llvmpasses/pipeline-o2.jl | 15 +++ test/llvmpasses/pipeline-prints.ll | 13 +-- 6 files changed, 118 insertions(+), 73 deletions(-) diff --git a/src/pipeline.cpp b/src/pipeline.cpp index d0456844e92c61..aaadc6674702df 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -24,11 +24,13 @@ #include // NewPM needs to manually include all the pass headers +#include #include #include #include #include #include +#include #include #include #include @@ -36,6 +38,8 @@ #include #include #include +#include +#include "llvm/Transforms/Scalar/ConstraintElimination.h" #include #include #include @@ -59,13 +63,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include +#include #include #include #include @@ -196,10 +204,9 @@ namespace { .convertSwitchRangeToICmp(true) .convertSwitchToLookupTable(true) .forwardSwitchCondToPhi(true) - //These mess with loop rotation, so only do them after that + .needCanonicalLoops(false) .hoistCommonInsts(true) - // Causes an SRET assertion error in late-gc-lowering - // .sinkCommonInsts(true) + .sinkCommonInsts(true) ; } @@ -341,10 +348,16 @@ static void buildEarlySimplificationPipeline(ModulePassManager &MPM, PassBuilder FPM.addPass(DCEPass()); FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions())); if (O.getSpeedupLevel() >= 1) { - // TODO check the LLVM 15 default. - FPM.addPass(SROAPass(SROAOptions::PreserveCFG)); + FPM.addPass(SROAPass(SROAOptions::ModifyCFG)); + FPM.addPass(EarlyCSEPass()); } MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + if (O.getSpeedupLevel() >= 1) { + FunctionPassManager GlobalFPM; + MPM.addPass(GlobalOptPass()); + GlobalFPM.addPass(PromotePass()); + GlobalFPM.addPass(InstCombinePass()); + } } invokeEarlySimplificationCallbacks(MPM, PB, O); } @@ -379,22 +392,24 @@ static void buildEarlyOptimizerPipeline(ModulePassManager &MPM, PassBuilder *PB, if (O.getSpeedupLevel() >= 1) { FunctionPassManager FPM; if (O.getSpeedupLevel() >= 2) { - // TODO check the LLVM 15 default. - FPM.addPass(SROAPass(SROAOptions::PreserveCFG)); - // SROA can duplicate PHI nodes which can block LowerSIMD - FPM.addPass(InstCombinePass()); - FPM.addPass(JumpThreadingPass()); - FPM.addPass(CorrelatedValuePropagationPass()); - FPM.addPass(ReassociatePass()); - FPM.addPass(EarlyCSEPass()); - JULIA_PASS(FPM.addPass(AllocOptPass())); - } else { // if (O.getSpeedupLevel() >= 1) (exactly) - FPM.addPass(InstCombinePass()); - FPM.addPass(EarlyCSEPass()); - } - invokePeepholeEPCallbacks(FPM, PB, O); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + FPM.addPass(SROAPass(SROAOptions::ModifyCFG)); + FPM.addPass(EarlyCSEPass(true)); + FPM.addPass(InstCombinePass()); + FPM.addPass(AggressiveInstCombinePass()); + FPM.addPass(JumpThreadingPass()); + FPM.addPass(CorrelatedValuePropagationPass()); + FPM.addPass(LibCallsShrinkWrapPass()); + FPM.addPass(ReassociatePass()); + FPM.addPass(ConstraintEliminationPass()); + JULIA_PASS(FPM.addPass(AllocOptPass())); + } else { // if (O.getSpeedupLevel() >= 1) (exactly) + FPM.addPass(EarlyCSEPass()); + FPM.addPass(InstCombinePass()); + } + invokePeepholeEPCallbacks(FPM, PB, O); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), /*UseMemorySSA = */true)); } + MPM.addPass(GlobalOptPass()); MPM.addPass(GlobalDCEPass()); } MPM.addPass(AfterEarlyOptimizationMarkerPass()); @@ -407,34 +422,32 @@ static void buildLoopOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB LoopPassManager LPM; LPM.addPass(LowerSIMDLoopPass()); if (O.getSpeedupLevel() >= 2) { - LPM.addPass(LoopRotatePass()); + LPM.addPass(LoopInstSimplifyPass()); + LPM.addPass(LoopSimplifyCFGPass()); + LPM.addPass(BeforeLICMMarkerPass()); + auto opts = LICMOptions(); + opts.AllowSpeculation = false; + LPM.addPass(LICMPass(opts)); + LPM.addPass(JuliaLICMPass()); + LPM.addPass(LoopRotatePass(true, false)); + LPM.addPass(LICMPass(LICMOptions())); + LPM.addPass(JuliaLICMPass()); + LPM.addPass(AfterLICMMarkerPass()); + LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true)); } invokeLateLoopOptimizationCallbacks(LPM, PB, O); //We don't know if the loop callbacks support MSSA - FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false)); - } - if (O.getSpeedupLevel() >= 2) { - LoopPassManager LPM; - LPM.addPass(BeforeLICMMarkerPass()); - LPM.addPass(LICMPass(LICMOptions())); - LPM.addPass(JuliaLICMPass()); - LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true)); - LPM.addPass(LICMPass(LICMOptions())); - LPM.addPass(JuliaLICMPass()); - LPM.addPass(AfterLICMMarkerPass()); - //LICM needs MemorySSA now, so we must use it FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */true)); } - if (O.getSpeedupLevel() >= 2) { + if (O.getSpeedupLevel() >= 2) FPM.addPass(IRCEPass()); - } { LoopPassManager LPM; LPM.addPass(BeforeLoopSimplificationMarkerPass()); if (O.getSpeedupLevel() >= 2) { - LPM.addPass(LoopInstSimplifyPass()); LPM.addPass(LoopIdiomRecognizePass()); LPM.addPass(IndVarSimplifyPass()); + LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true)); LPM.addPass(LoopDeletionPass()); // This unroll will only unroll loops when the trip count is known and small, // so that no loop remains @@ -442,6 +455,8 @@ static void buildLoopOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB } invokeLoopOptimizerEndCallbacks(LPM, PB, O); LPM.addPass(AfterLoopSimplificationMarkerPass()); + FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions())); + FPM.addPass(InstCombinePass()); //We don't know if the loop end callbacks support MSSA FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false)); } @@ -454,17 +469,28 @@ static void buildScalarOptimizerPipeline(FunctionPassManager &FPM, PassBuilder * if (options.enable_scalar_optimizations) { if (O.getSpeedupLevel() >= 2) { JULIA_PASS(FPM.addPass(AllocOptPass())); - // TODO check the LLVM 15 default. - FPM.addPass(SROAPass(SROAOptions::PreserveCFG)); - FPM.addPass(InstSimplifyPass()); + FPM.addPass(SROAPass(SROAOptions::ModifyCFG)); + FPM.addPass(VectorCombinePass(/*TryEarlyFoldsOnly=*/true)); + FPM.addPass(MergedLoadStoreMotionPass()); FPM.addPass(GVNPass()); - FPM.addPass(MemCpyOptPass()); FPM.addPass(SCCPPass()); + FPM.addPass(BDCEPass()); + FPM.addPass(InstCombinePass()); FPM.addPass(CorrelatedValuePropagationPass()); - FPM.addPass(DCEPass()); + FPM.addPass(ADCEPass()); + FPM.addPass(MemCpyOptPass()); + FPM.addPass(DSEPass()); FPM.addPass(IRCEPass()); - FPM.addPass(InstCombinePass()); FPM.addPass(JumpThreadingPass()); + FPM.addPass(ConstraintEliminationPass()); + } else if (O.getSpeedupLevel() >= 1) { + JULIA_PASS(FPM.addPass(AllocOptPass())); + FPM.addPass(SROAPass(SROAOptions::ModifyCFG)); + FPM.addPass(MemCpyOptPass()); + FPM.addPass(SCCPPass()); + FPM.addPass(BDCEPass()); + FPM.addPass(InstCombinePass()); + FPM.addPass(ADCEPass()); } if (O.getSpeedupLevel() >= 3) { FPM.addPass(GVNPass()); @@ -476,12 +502,15 @@ static void buildScalarOptimizerPipeline(FunctionPassManager &FPM, PassBuilder * JULIA_PASS(FPM.addPass(AllocOptPass())); { LoopPassManager LPM; - LPM.addPass(LoopDeletionPass()); - LPM.addPass(LoopInstSimplifyPass()); - FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM))); + LPM.addPass(LICMPass(LICMOptions())); + LPM.addPass(JuliaLICMPass()); + FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */true)); } - FPM.addPass(LoopDistributePass()); - } + FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); + FPM.addPass(InstCombinePass()); + } else if (O.getSpeedupLevel() >= 1) + FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); + invokeScalarOptimizerCallbacks(FPM, PB, O); } FPM.addPass(AfterScalarOptimizationMarkerPass()); @@ -491,19 +520,27 @@ static void buildVectorPipeline(FunctionPassManager &FPM, PassBuilder *PB, Optim FPM.addPass(BeforeVectorizationMarkerPass()); if (options.enable_vector_pipeline) { //TODO look into loop vectorize options + // Rerotate loops that might have been unrotated in the simplification + LoopPassManager LPM; + LPM.addPass(LoopRotatePass()); + LPM.addPass(LoopDeletionPass()); + FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false)); + FPM.addPass(LoopDistributePass()); FPM.addPass(InjectTLIMappings()); FPM.addPass(LoopVectorizePass()); FPM.addPass(LoopLoadEliminationPass()); - FPM.addPass(InstCombinePass()); FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); + FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(LICMOptions()), /*UseMemorySSA=*/true, /*UseBlockFrequencyInfo=*/false)); + FPM.addPass(EarlyCSEPass()); + FPM.addPass(CorrelatedValuePropagationPass()); + FPM.addPass(InstCombinePass()); FPM.addPass(SLPVectorizerPass()); - invokeVectorizerCallbacks(FPM, PB, O); FPM.addPass(VectorCombinePass()); - FPM.addPass(ADCEPass()); - //TODO add BDCEPass here? - // This unroll will unroll vectorized loops - // as well as loops that we tried but failed to vectorize + invokeVectorizerCallbacks(FPM, PB, O); FPM.addPass(LoopUnrollPass(LoopUnrollOptions(O.getSpeedupLevel(), /*OnlyWhenForced = */ false, /*ForgetSCEV = */false))); + FPM.addPass(SROAPass(SROAOptions::PreserveCFG)); + FPM.addPass(InstSimplifyPass()); + FPM.addPass(AfterVectorizationMarkerPass()); } FPM.addPass(AfterVectorizationMarkerPass()); } @@ -525,18 +562,18 @@ static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder * FunctionPassManager FPM; JULIA_PASS(FPM.addPass(LateLowerGCPass())); JULIA_PASS(FPM.addPass(FinalLowerGCPass())); - if (O.getSpeedupLevel() >= 2) { - FPM.addPass(DSEPass()); - FPM.addPass(GVNPass()); - FPM.addPass(SCCPPass()); - FPM.addPass(DCEPass()); - } MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native))); MPM.addPass(RemoveJuliaAddrspacesPass()); //TODO: Make this conditional on arches (GlobalISel doesn't like our addrsspaces) if (O.getSpeedupLevel() >= 1) { FunctionPassManager FPM; + if (O.getSpeedupLevel() >= 2) { + FPM.addPass(DSEPass()); + FPM.addPass(GVNPass()); + FPM.addPass(SCCPPass()); + FPM.addPass(DCEPass()); + } FPM.addPass(InstCombinePass()); FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index 3b2f8539992290..af57c8ee6c1566 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -299,7 +299,7 @@ end |> only === Type{Int} if bc_opt == bc_default # Array/Memory escape analysis - function no_allocate(T::Type{<:Union{Memory, Vector}}) + function no_allocate(T::Type{<:Union{Memory}}) v = T(undef, 2) v[1] = 2 v[2] = 3 @@ -308,7 +308,7 @@ if bc_opt == bc_default function test_alloc(::Type{T}; broken=false) where T @test (@allocated no_allocate(T)) == 0 broken=broken end - for T in [Memory, Vector] + for T in [Memory] # This requires changing the pointer_from_objref to something llvm sees through for ET in [Int, Float32, Union{Int, Float64}] no_allocate(T{ET}) #compile # allocations aren't removed for Union eltypes which they theoretically could be eventually diff --git a/test/llvmpasses/image-codegen.jl b/test/llvmpasses/image-codegen.jl index 2e52245b7d3b94..35e5add2de601d 100644 --- a/test/llvmpasses/image-codegen.jl +++ b/test/llvmpasses/image-codegen.jl @@ -14,8 +14,7 @@ # CHECK-NOT: private global # CHECK: jl_global # COM: we emit both declarations and definitions, so we may see either style in the IR -# CHECK-SAME: = {{(external )?}}global -# CHECK: julia_f_ +# CHECK-SAME: = {{(external )?}} # CHECK-NOT: internal global # CHECK-NOT: private global diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index 81a1df61d3bd92..4739fa186ffc7b 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -207,3 +207,4 @@ define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { ; CHECK-NEXT: !10 = distinct !{!10} ; CHECK-NEXT: !11 = !{!12, !12, i64 0} ; CHECK-NEXT: !12 = !{!"jtbaa_const", !3} + diff --git a/test/llvmpasses/pipeline-o2.jl b/test/llvmpasses/pipeline-o2.jl index ceb2fe3bf65d6a..ea2fa293c7ebcf 100644 --- a/test/llvmpasses/pipeline-o2.jl +++ b/test/llvmpasses/pipeline-o2.jl @@ -131,6 +131,18 @@ function loopedlength(arr) end len end +# COM: Vector +# ALL-LABEL: @julia_memset_like +# ALL: vector.body + +# COM: Memory +# ALL-LABEL: @julia_memset_like +# ALL: vector.body +function memset_like(mem) + for idx in eachindex(mem) + mem[idx] = 1.0 + end +end emit(iterate_read, Vector{Int64}) emit(iterate_write, Vector{Int64}, Vector{Int64}) @@ -150,3 +162,6 @@ emit(sumloop, Int64) emit(simd_sumloop, Float32) emit(loopedlength, Vector{Int64}) + +emit(memset_like, Vector{Float64}) +emit(memset_like, Memory{Float64}) diff --git a/test/llvmpasses/pipeline-prints.ll b/test/llvmpasses/pipeline-prints.ll index 9c27885c5ca454..9b5732981faf63 100644 --- a/test/llvmpasses/pipeline-prints.ll +++ b/test/llvmpasses/pipeline-prints.ll @@ -285,25 +285,18 @@ attributes #2 = { inaccessiblemem_or_argmemonly } ; COM: InstSimplify/InstCombine should kill this zext-trunc pair ; AFTEREARLYSIMPLIFICATION: [[ZEXT:%.*]] = zext i1 {{%.*}} to i8 -; AFTEREARLYSIMPLIFICATION-NEXT: trunc i8 [[ZEXT]] to i1 - -; BEFOREEARLYOPTIMIZATION: [[ZEXT:%.*]] = zext i1 {{%.*}} to i8 -; BEFOREEARLYOPTIMIZATION-NEXT: trunc i8 [[ZEXT]] to i1 ; AFTEREARLYOPTIMIZATION-NOT: zext i1 {{%.*}} to i8 -; AFTEREARLYOPTIMIZATION-NOT: trunc i8 {{%.*}} to i1 ; BEFORELOOPOPTIMIZATION-NOT: zext i1 {{%.*}} to i8 -; BEFORELOOPOPTIMIZATION-NOT: trunc i8 {{%.*}} to i1 ; COM: Loop simplification makes the exit condition obvious ; AFTERLOOPSIMPLIFICATION: L35.lr.ph: ; AFTERLOOPSIMPLIFICATION: add nuw nsw -; COM: Scalar optimization removes the previous add from the preheader -; AFTERSCALAROPTIMIZATION: L35.lr.ph: -; AFTERSCALAROPTIMIZATION-NOT: add nuw nsw -; AFTERSCALAROPTIMIZATION: br label %L35 +; COM: Scalar optimization removes the preheader +; AFTERSCALAROPTIMIZATION: L17: +; AFTERSCALAROPTIMIZATION: icmp eq i64 {{%.*}}, 1, ; COM: Vectorization does stuff ; AFTERVECTORIZATION: vector.body From e026affc3ad49ac199cf22ce05aeb412d50fcb21 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 21 Feb 2025 10:00:47 -0500 Subject: [PATCH 14/25] build: fix makefile rules and windows path separator used in build (#57472) Closes #45262 --- Make.inc | 81 +++++++++++++++++++++++------------------ Makefile | 16 ++++---- base/Makefile | 87 +++++++++++++++++++------------------------- cli/Makefile | 4 +- src/Makefile | 8 ++-- src/flisp/Makefile | 8 ++-- src/support/Makefile | 8 ++-- 7 files changed, 106 insertions(+), 106 deletions(-) diff --git a/Make.inc b/Make.inc index 16e238c6f0683c..466e97cc43c4f6 100644 --- a/Make.inc +++ b/Make.inc @@ -111,6 +111,11 @@ endef COMMA:=, SPACE:=$(eval) $(eval) +# define various helper macros for safe interpolation into various parsers +shell_escape='$(subst ','\'',$1)' +c_escape="$(subst ",\",$(subst \,\\,$1))" +julia_escape=$(call c_escape,$1) + # force a sane / stable configuration export LC_ALL=C export LANG=C @@ -644,12 +649,12 @@ CXX += -Qunused-arguments export CCACHE_CPP2 := yes endif else #USECCACHE -CC_BASE := $(shell echo $(CC) | cut -d' ' -f1) -CC_ARG := $(shell echo $(CC) | cut -s -d' ' -f2-) -CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1) -CXX_ARG := $(shell echo $(CXX) | cut -s -d' ' -f2-) -FC_BASE := $(shell echo $(FC) 2>/dev/null | cut -d' ' -f1) -FC_ARG := $(shell echo $(FC) 2>/dev/null | cut -s -d' ' -f2-) +CC_BASE := $(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -d' ' -f1) +CC_ARG := $(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -s -d' ' -f2-) +CXX_BASE := $(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -d' ' -f1) +CXX_ARG := $(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -s -d' ' -f2-) +FC_BASE := $(shell printf "%s\n" $(call shell_escape,$(FC)) 2>/dev/null | cut -d' ' -f1) +FC_ARG := $(shell printf "%s\n" $(call shell_escape,$(FC)) 2>/dev/null | cut -s -d' ' -f2-) endif JFFLAGS := -O2 $(fPIC) @@ -776,7 +781,7 @@ LDFLAGS += -L$(build_libdir) -Wl,-rpath,$(build_libdir) endif # gfortran endif # FreeBSD -ifneq ($(CC_BASE)$(CXX_BASE),$(shell echo $(CC) | cut -d' ' -f1)$(shell echo $(CXX) | cut -d' ' -f1)) +ifneq ($(CC_BASE)$(CXX_BASE),$(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -d' ' -f1)$(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -d' ' -f1)) $(error Forgot override directive on CC or CXX in Make.user? Cowardly refusing to build) endif @@ -1663,6 +1668,12 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1))) endef endif +ifeq ($(OS), WINNT) +normalize_path = $(subst /,\,$1) +else +normalize_path = $1 +endif + define symlink_target # (from, to-dir, to-name) CLEAN_TARGETS += clean-$$(abspath $(2)/$(3)) clean-$$(abspath $(2)/$(3)): @@ -1729,20 +1740,20 @@ JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT) JULIA_SYSIMG := $(JULIA_SYSIMG_$(JULIA_BUILD_MODE)) define dep_lib_path -$(shell $(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2)) +$(call normalize_path,$(shell $(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2))) endef -LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) ifeq ($(OS),WINNT) ifeq ($(BINARY),32) @@ -1770,34 +1781,34 @@ endif # USE_SYSTEM_CSL causes it to get symlinked into build_private_shlibdir ifeq ($(USE_SYSTEM_CSL),1) -LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBGCC_NAME)) else -LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBGCC_NAME)) endif -LIBGCC_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBGCC_NAME)) # We only bother to define this on Linux, as that's the only platform that does libstdc++ probing # On all other platforms, the LIBSTDCXX_*_DEPLIB variables will be empty. ifeq ($(OS),Linux) LIBSTDCXX_NAME := libstdc++.so.6 ifeq ($(USE_SYSTEM_CSL),1) -LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBSTDCXX_NAME)) else -LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBSTDCXX_NAME)) endif -LIBSTDCXX_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBSTDCXX_NAME)) endif # USE_SYSTEM_LIBM and USE_SYSTEM_OPENLIBM causes it to get symlinked into build_private_shlibdir ifeq ($(USE_SYSTEM_LIBM),1) -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) else ifeq ($(USE_SYSTEM_OPENLIBM),1) -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) else -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) endif -LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) # We list: # * libgcc_s, because FreeBSD needs to load ours, not the system one. @@ -1861,7 +1872,7 @@ ifeq ($(VERBOSE), 0) QUIET_MAKE = -s -GOAL=$(subst ','\'',$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) +GOAL=$(call shell_escape,$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) PRINT_CC = printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$(GOAL)$(ENDCOLOR); $(1) PRINT_ANALYZE = printf ' %b %b\n' $(CCCOLOR)ANALYZE$(ENDCOLOR) $(SRCCOLOR)$(GOAL)$(ENDCOLOR); $(1) @@ -1873,13 +1884,13 @@ PRINT_DTRACE = printf ' %b %b\n' $(DTRACECOLOR)DTRACE$(ENDCOLOR) $(BINCOLOR)$ else QUIET_MAKE = -PRINT_CC = echo '$(subst ','\'',$(1))'; $(1) -PRINT_ANALYZE = echo '$(subst ','\'',$(1))'; $(1) -PRINT_LINK = echo '$(subst ','\'',$(1))'; $(1) -PRINT_PERL = echo '$(subst ','\'',$(1))'; $(1) -PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1) -PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1) -PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1) +PRINT_CC = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_ANALYZE = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_LINK = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_PERL = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_FLISP = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_JULIA = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_DTRACE = printf "%s\n" $(call shell_escape,$(1)); $(1) endif # VERBOSE @@ -1887,4 +1898,4 @@ endif # VERBOSE # call print-VARIABLE to see the runtime value of any variable # (hardened against any special characters appearing in the output) print-%: - @echo '$*=$(subst ','\'',$(subst $(newline),\n,$($*)))' + @printf "%s\n" $(call shell_escape,$*)=$(call shell_escape,$(subst $(newline),\n,$($*))) diff --git a/Makefile b/Makefile index 0f1e8c45edf401..a7562946c1699f 100644 --- a/Makefile +++ b/Makefile @@ -4,18 +4,18 @@ include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/llvm-ver.make # Make sure the user didn't try to build in a path that will confuse the shell or make -METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\":]\|/\./\|/\.\./ +METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\": ]\|/\./\|/\.\./ ifneq (,$(findstring ',$(value BUILDROOT))) $(error cowardly refusing to build into directory with a single-quote in the path) endif ifneq (,$(findstring ',$(value JULIAHOME))) $(error cowardly refusing to build from source directory with a single-quote in the path) endif -ifneq (,$(shell echo '$(value BUILDROOT)/' | grep '$(METACHARACTERS)')) +ifneq (,$(shell printf "%s\n" $(call shell_escape,$(value BUILDROOT)/) | grep '$(METACHARACTERS)')) $(error cowardly refusing to build into directory with a shell-metacharacter in the path\ (got: $(value BUILDROOT))) endif -ifneq (,$(shell echo '$(value JULIAHOME)/' | grep '$(METACHARACTERS)')) +ifneq (,$(shell printf "%s\n" $(call shell_escape,$(value JULIAHOME)/) | grep '$(METACHARACTERS)')) $(error cowardly refusing to build from source directory with a shell-metacharacter in the path\ (got: $(value JULIAHOME))) endif @@ -33,9 +33,9 @@ BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk $(B DIRS += $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ - @echo 'BUILDROOT=$(BUILDROOT)' >> $@ - @echo 'include $(JULIAHOME)$(patsubst $(BUILDROOT)%,%,$@)' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/Makefile -- #' > $@ + @printf "%s\n" 'BUILDROOT=$(BUILDROOT)' >> $@ + @printf "%s\n" 'include $(JULIAHOME)$(patsubst $(BUILDROOT)%,%,$@)' >> $@ julia-deps: | $(BUILDDIRMAKE) configure-y: | $(BUILDDIRMAKE) configure: @@ -67,7 +67,7 @@ $(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUI julia-symlink: julia-cli-$(JULIA_BUILD_MODE) ifeq ($(OS),WINNT) - echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat + printf '@"%%~dp0/%s" %%*\n' "$$(printf "%s\n" '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')" | tr / '\\' > $(BUILDROOT)/julia.bat chmod a+x $(BUILDROOT)/julia.bat else ifndef JULIA_VAGRANT_BUILD @@ -306,7 +306,7 @@ endif # Note that we disable MSYS2's path munging here, as otherwise # it replaces our `:`-separated list as a `;`-separated one. define stringreplace - MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep "$2" | awk '{print $$1;}') "$3" 255 "$(call cygpath_w,$1)" + MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep '$2' | awk '{print $$1;}') '$3' 255 '$(call cygpath_w,$1)' endef diff --git a/base/Makefile b/base/Makefile index 09f79e5b98611e..34791f7b4b0d4f 100644 --- a/base/Makefile +++ b/base/Makefile @@ -18,9 +18,9 @@ else endif define parse_features -@echo "# $(2) features" >> $@ +@printf "%s\n" "# $(2) features" >> $@ @$(call PRINT_PERL, cat $(SRCDIR)/../src/features_$(1).h | perl -lne 'print "const JL_$(2)_$$1 = UInt32($$2)" if /^\s*JL_FEATURE_DEF(?:_NAME)?\(\s*(\w+)\s*,\s*([^,]+)\s*,.*\)\s*(?:\/\/.*)?$$/' >> $@) -@echo >> $@ +@printf "\n" >> $@ endef $(BUILDDIR)/features_h.jl: $(SRCDIR)/../src/features_x86.h $(SRCDIR)/../src/features_aarch32.h $(SRCDIR)/../src/features_aarch64.h @@ -33,7 +33,7 @@ $(BUILDDIR)/pcre_h.jl: $(PCRE_INCL_PATH) @$(call PRINT_PERL, $(CPP) -D PCRE2_CODE_UNIT_WIDTH=8 -dM $< | perl -nle '/^\s*#define\s+PCRE2_(\w*)\s*\(?($(PCRE_CONST))\)?u?\s*$$/ and print index($$1, "ERROR_") == 0 ? "const $$1 = Cint($$2)" : "const $$1 = UInt32($$2)"' | LC_ALL=C sort > $@) $(BUILDDIR)/errno_h.jl: - @$(call PRINT_PERL, echo '#include ' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | LC_ALL=C sort > $@) + @$(call PRINT_PERL, printf "%s\n" '#include ' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | LC_ALL=C sort > $@) $(BUILDDIR)/file_constants.jl: $(SRCDIR)/../src/file_constants.h @$(call PRINT_PERL, $(CPP_STDOUT) -DJULIA $< | perl -nle 'print "$$1 0o$$2" if /^(\s*const\s+[A-z_]+\s+=)\s+(0[0-9]*)\s*$$/; print "$$1" if /^\s*(const\s+[A-z_]+\s+=\s+([1-9]|0x)[0-9A-z]*)\s*$$/' > $@) @@ -42,57 +42,46 @@ $(BUILDDIR)/uv_constants.jl: $(SRCDIR)/../src/uv_constants.h $(LIBUV_INC)/uv/err @$(call PRINT_PERL, $(CPP_STDOUT) "-I$(LIBUV_INC)" -DJULIA $< | tail -n 16 > $@) $(BUILDDIR)/build_h.jl.phony: - @echo "# This file is automatically generated in base/Makefile" > $@ + @printf "%s\n" "# This file is automatically generated in base/Makefile" > $@ ifeq ($(XC_HOST),) - @echo "const MACHINE = \"$(BUILD_MACHINE)\"" >> $@ + @printf "%s\n" "const MACHINE = \"$(BUILD_MACHINE)\"" >> $@ else - @echo "const MACHINE = \"$(XC_HOST)\"" >> $@ + @printf "%s\n" "const MACHINE = \"$(XC_HOST)\"" >> $@ endif - @echo "const libm_name = \"$(LIBMNAME)\"" >> $@ + @printf "%s\n" "const libm_name = \"$(LIBMNAME)\"" >> $@ ifeq ($(USE_BLAS64), 1) - @echo "const USE_BLAS64 = true" >> $@ + @printf "%s\n" "const USE_BLAS64 = true" >> $@ else - @echo "const USE_BLAS64 = false" >> $@ + @printf "%s\n" "const USE_BLAS64 = false" >> $@ endif ifeq ($(USE_GPL_LIBS), 1) - @echo "const USE_GPL_LIBS = true" >> $@ + @printf "%s\n" "const USE_GPL_LIBS = true" >> $@ else - @echo "const USE_GPL_LIBS = false" >> $@ -endif - @echo "const libllvm_version_string = \"$$($(LLVM_CONFIG_HOST) --version)\"" >> $@ - @echo "const libllvm_name = \"$(LLVM_SHARED_LIB_NAME)\"" >> $@ - @echo "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@ - @echo "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@ -ifeq ($(OS),WINNT) - @printf 'const SYSCONFDIR = "%s"\n' '$(subst /,\\,$(sysconfdir_rel))' >> $@ - @printf 'const DATAROOTDIR = "%s"\n' '$(subst /,\\,$(datarootdir_rel))' >> $@ - @printf 'const DOCDIR = "%s"\n' '$(subst /,\\,$(docdir_rel))' >> $@ - @printf 'const LIBDIR = "%s"\n' '$(subst /,\\,$(libdir_rel))' >> $@ - @printf 'const LIBEXECDIR = "%s"\n' '$(subst /,\\,$(libexecdir_rel))' >> $@ - @printf 'const PRIVATE_LIBDIR = "%s"\n' '$(subst /,\\,$(private_libdir_rel))' >> $@ - @printf 'const PRIVATE_LIBEXECDIR = "%s"\n' '$(subst /,\\,$(private_libexecdir_rel))' >> $@ - @printf 'const INCLUDEDIR = "%s"\n' '$(subst /,\\,$(includedir_rel))' >> $@ -else - @echo "const SYSCONFDIR = \"$(sysconfdir_rel)\"" >> $@ - @echo "const DATAROOTDIR = \"$(datarootdir_rel)\"" >> $@ - @echo "const DOCDIR = \"$(docdir_rel)\"" >> $@ - @echo "const LIBDIR = \"$(libdir_rel)\"" >> $@ - @echo "const LIBEXECDIR = \"$(libexecdir_rel)\"" >> $@ - @echo "const PRIVATE_LIBDIR = \"$(private_libdir_rel)\"" >> $@ - @echo "const PRIVATE_LIBEXECDIR = \"$(private_libexecdir_rel)\"" >> $@ - @echo "const INCLUDEDIR = \"$(includedir_rel)\"" >> $@ + @printf "%s\n" "const USE_GPL_LIBS = false" >> $@ endif + @printf "%s\n" "const libllvm_version_string = \"$$($(LLVM_CONFIG_HOST) --version)\"" >> $@ + @printf "%s\n" "const libllvm_name = \"$(LLVM_SHARED_LIB_NAME)\"" >> $@ + @printf "%s\n" "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@ + @printf "%s\n" "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@ + @printf "%s\n" "const SYSCONFDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(sysconfdir_rel)))) >> $@ + @printf "%s\n" "const DATAROOTDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(datarootdir_rel)))) >> $@ + @printf "%s\n" "const DOCDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(docdir_rel)))) >> $@ + @printf "%s\n" "const LIBDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(libdir_rel)))) >> $@ + @printf "%s\n" "const LIBEXECDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(libexecdir_rel)))) >> $@ + @printf "%s\n" "const PRIVATE_LIBDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libdir_rel)))) >> $@ + @printf "%s\n" "const PRIVATE_LIBEXECDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libexecdir_rel)))) >> $@ + @printf "%s\n" "const INCLUDEDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(includedir_rel)))) >> $@ ifeq ($(DARWIN_FRAMEWORK), 1) - @echo "const DARWIN_FRAMEWORK = true" >> $@ - @echo "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK = true" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ else - @echo "const DARWIN_FRAMEWORK = false" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK = false" >> $@ endif ifeq ($(OS), Darwin) - @echo "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ - @echo "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ + @printf "%s\n" "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ + @printf "%s\n" "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ endif - @echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ + @printf "%s\n" "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ @# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl, @# ensuring we rebuild the system image as infrequently as possible @@ -115,10 +104,10 @@ ifneq ($(NO_GIT), 1) rm -f $@; \ fi else -ifeq ($(shell [ -f $(BUILDDIR)/version_git.jl ] && echo "true"), true) +ifeq ($(shell [ -f $(BUILDDIR)/version_git.jl ] && printf "true\n"), true) @# Give warning if boilerplate git is used @if grep -q "Default output if git is not available" $(BUILDDIR)/version_git.jl; then \ - echo "WARNING: Using boilerplate git version info" >&2; \ + printf "WARNING: Using boilerplate git version info\n" >&2; \ fi else $(warning "WARNING: Generating boilerplate git version info") @@ -141,7 +130,7 @@ resolve_path = \ if [ -n "$${$1_}" ]; then $1_wd=`dirname "$${$1}"`; $1="$${$1_}"; fi ## if it's a relative path, make it an absolute path resolve_path += && \ - if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi + if [ -z "`printf "%s\n" "$${$1}" | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi ifeq ($(OS), Darwin) # try to use the install_name id instead (unless it is an @rpath or such) # if it's a relative path, make it an absolute path using the working directory from $1, @@ -150,7 +139,7 @@ resolve_path += && \ $1_=`otool -D $${$1} | tail -n +2 | sed -e 's/^@.*$$//'` && \ if [ -n "$${$1_}" ]; then \ $1_wd=`dirname "$${$1}"`; $1=$${$1_}; \ - if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi; \ + if [ -z "`printf "%s\n" $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi; \ fi else # try to use the SO_NAME (if the named file exists) @@ -164,10 +153,10 @@ endif ## debug code: `make resolve-path P=` #resolve_path += && \ -# echo "$${$1_wd} $${$1}" +# printf "%s\n" "$${$1_wd} $${$1}" #resolve-path: # $(call resolve_path,P) && \ -# echo "$$P" +# printf "%s\n" "$$P" define symlink_system_library libname_$2 := $$(notdir $(call versioned_libname,$2,$3)) @@ -179,11 +168,11 @@ $$(build_private_libdir)/$$(libname_$2): $$(call resolve_path,REALPATH) && \ [ -e "$$$$REALPATH" ] && \ rm -f "$$@" && \ - echo ln -sf "$$$$REALPATH" "$$@" && \ + printf "ln -sf %s %s\n" "$$$$REALPATH" "$$@" && \ ln -sf "$$$$REALPATH" "$$@"; \ else \ if [ "$4" != "ALLOW_FAILURE" ]; then \ - echo "System library symlink failure: Unable to locate $$(libname_$2) on your system!" >&2; \ + printf "%s\n" "System library symlink failure: Unable to locate $$(libname_$2) on your system!" >&2; \ false; \ fi; \ fi @@ -295,7 +284,7 @@ $(build_private_libdir)/libLLVM.$(SHLIB_EXT): $(call resolve_path,REALPATH) && \ [ -e "$$REALPATH" ] && \ rm -f "$@" && \ - echo ln -sf "$$REALPATH" "$@" && \ + printf "%s\n" ln -sf "$$REALPATH" "$@" && \ ln -sf "$$REALPATH" "$@" ifneq ($(USE_SYSTEM_LLVM),0) ifneq ($(USE_LLVM_SHLIB),0) diff --git a/cli/Makefile b/cli/Makefile index 3cc0af1a76afd5..8c73d76f5020fd 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -28,8 +28,8 @@ LOADER_LDFLAGS += -Wl,--no-as-needed -lpthread -rdynamic -lc -Wl,--as-needed endif # Build list of dependent libraries that must be opened -SHIPFLAGS += -DDEP_LIBS="\"$(LOADER_BUILD_DEP_LIBS)\"" -DEBUGFLAGS += -DDEP_LIBS="\"$(LOADER_DEBUG_BUILD_DEP_LIBS)\"" +SHIPFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_BUILD_DEP_LIBS))) +DEBUGFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_DEBUG_BUILD_DEP_LIBS))) ifneq (,$(findstring MINGW,$(shell uname))) # In MSYS2, do not perform path conversion for `DEP_LIBS`. # https://www.msys2.org/wiki/Porting/#filesystem-namespaces diff --git a/src/Makefile b/src/Makefile index 130c35960f2d0c..5c5e19217bd75e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -205,10 +205,10 @@ CODEGEN_OBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.o) CODEGEN_DOBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.dbg.obj) # Add SONAME defines so we can embed proper `dlopen()` calls. -ADDL_SHIPFLAGS := "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.$(SHLIB_EXT)\"" \ - "-DJL_LIBJULIA_SONAME=\"$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT)\"" -ADDL_DEBUGFLAGS := "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys-debug.$(SHLIB_EXT)\"" \ - "-DJL_LIBJULIA_SONAME=\"$(LIBJULIA_PATH_REL)-debug.$(JL_MAJOR_SHLIB_EXT)\"" +ADDL_SHIPFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys.$(SHLIB_EXT)))) \ + -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT))) +ADDL_DEBUGFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys-debug.$(SHLIB_EXT)))) \ + -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL)-debug.$(JL_MAJOR_SHLIB_EXT))) SHIPFLAGS += $(FLAGS) $(ADDL_SHIPFLAGS) DEBUGFLAGS += $(FLAGS) $(ADDL_DEBUGFLAGS) diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 17292d301115b7..eca1de86e588a5 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -111,10 +111,10 @@ $(BUILDDIR)/$(EXENAME)$(EXE): $(OBJS) $(LIBFILES_release) $(BUILDDIR)/$(LIBTARGE $(BUILDDIR)/host/Makefile: mkdir -p $(BUILDDIR)/host @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/src/flisp/Makefile -- #' > $@ - @echo 'BUILDDIR=$(BUILDDIR)/host' >> $@ - @echo 'BUILDING_HOST_TOOLS=1' >> $@ - @echo 'include $(SRCDIR)/Makefile' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/src/flisp/Makefile -- #' > $@ + @printf "%s\n" 'BUILDDIR=$(BUILDDIR)/host' >> $@ + @printf "%s\n" 'BUILDING_HOST_TOOLS=1' >> $@ + @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ $(BUILDDIR)/host/$(EXENAME): $(BUILDDIR)/host/Makefile | ${BUILDDIR}/host/flisp.boot make -C $(BUILDDIR)/host $(EXENAME) diff --git a/src/support/Makefile b/src/support/Makefile index 1ee98a4eabdeec..c7de154058586f 100644 --- a/src/support/Makefile +++ b/src/support/Makefile @@ -48,10 +48,10 @@ $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.S | $(BUILDDIR) $(BUILDDIR)/host/Makefile: mkdir -p $(BUILDDIR)/host @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ - @echo 'BUILDDIR=$(BUILDDIR)/host' >> $@ - @echo 'BUILDING_HOST_TOOLS=1' >> $@ - @echo 'include $(SRCDIR)/Makefile' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/Makefile -- #' > $@ + @printf "%s\n" 'BUILDDIR=$(BUILDDIR)/host' >> $@ + @printf "%s\n" 'BUILDING_HOST_TOOLS=1' >> $@ + @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ release: $(BUILDDIR)/libsupport.a debug: $(BUILDDIR)/libsupport-debug.a From e2cc68ced04dbfa4ea2005a4a44e05fe1e6d6336 Mon Sep 17 00:00:00 2001 From: Tommy Hofmann Date: Fri, 21 Feb 2025 19:41:27 +0100 Subject: [PATCH 15/25] Clarify disabling of tab-completion hinting in the documentation (#57493) --- stdlib/REPL/docs/src/index.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/docs/src/index.md b/stdlib/REPL/docs/src/index.md index eabd7e729280ed..ddd0a0953fcfc3 100644 --- a/stdlib/REPL/docs/src/index.md +++ b/stdlib/REPL/docs/src/index.md @@ -343,7 +343,15 @@ mapfoldl mapfoldr When a single complete tab-complete result is available at the end of an input line and 2 or more characters have been typed, a hint of the completion will show in a lighter color. -This can be disabled via `Base.active_repl.options.hint_tab_completes = false`. +This can be disabled via `Base.active_repl.options.hint_tab_completes = false` or by adding +``` +atreplinit() do repl + if VERSION >= v"1.11.0-0" + repl.options.hint_tab_completes = false + end +end +``` +to your `~/.julia/config/startup.jl`. !!! compat "Julia 1.11" Tab-complete hinting was added in Julia 1.11 From 8b9a19f661d2b4505c5eef4310f7fa22aa56b61e Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:05:14 -0500 Subject: [PATCH 16/25] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Do?= =?UTF-8?q?wnloads=20stdlib=20from=20e692e77=20to=20e7fe58f=20(#57495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Downloads URL: https://github.com/JuliaLang/Downloads.jl.git Stdlib branch: master Julia branch: master Old commit: e692e77 New commit: e7fe58f Julia version: 1.13.0-DEV Downloads version: 1.7.0(It's okay that it doesn't match) Bump invoked by: @StefanKarpinski Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Downloads.jl/compare/e692e77fb5427bf3c6e81514b323c39a88217eec...e7fe58f330301f9ed9d6d5ded64df70e65d674c2 ``` $ git log --oneline e692e77..e7fe58f e7fe58f Oops: delete debug lines 9f686f0 Get default file name from content-disposition or URL (#269) 9391b09 Adapt code to more recent Julia versions 03f0d4d Drop support (and CI) for Julia < 1.10 (LTS) ``` Co-authored-by: StefanKarpinski <153596+StefanKarpinski@users.noreply.github.com> --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/Downloads.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/md5 delete mode 100644 deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/sha512 create mode 100644 deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/md5 create mode 100644 deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/sha512 diff --git a/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/md5 b/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/md5 deleted file mode 100644 index 221a62b1cf2315..00000000000000 --- a/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -cdaea923f7fa855409e8456159251f54 diff --git a/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/sha512 b/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/sha512 deleted file mode 100644 index b537ef2e9e1f66..00000000000000 --- a/deps/checksums/Downloads-e692e77fb5427bf3c6e81514b323c39a88217eec.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e893fbe079a433c3038b79c4c2998d1ae9abaf92ff74152820a67e97ffee6f7f052085a7108410cbb1a3bd8cc6670736b0827c8b0608cc31941251dd6500d36a diff --git a/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/md5 b/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/md5 new file mode 100644 index 00000000000000..0e61a70a5b4b39 --- /dev/null +++ b/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/md5 @@ -0,0 +1 @@ +3b97f25ac9fc1c8fed56bf0a91d5e6ab diff --git a/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/sha512 b/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/sha512 new file mode 100644 index 00000000000000..d3b57b3b5280d6 --- /dev/null +++ b/deps/checksums/Downloads-e7fe58f330301f9ed9d6d5ded64df70e65d674c2.tar.gz/sha512 @@ -0,0 +1 @@ +dc03b3a2768cb455e11c3e8bd506dec907a79da9f2b336df816d9cc5b937e1d45148c218111c2a1b8471412ae9ec38c0d056bdb718cbd7031953830356a44396 diff --git a/stdlib/Downloads.version b/stdlib/Downloads.version index 40004d8337091a..3be8fe9807eb3b 100644 --- a/stdlib/Downloads.version +++ b/stdlib/Downloads.version @@ -1,4 +1,4 @@ DOWNLOADS_BRANCH = master -DOWNLOADS_SHA1 = e692e77fb5427bf3c6e81514b323c39a88217eec +DOWNLOADS_SHA1 = e7fe58f330301f9ed9d6d5ded64df70e65d674c2 DOWNLOADS_GIT_URL := https://github.com/JuliaLang/Downloads.jl.git DOWNLOADS_TAR_URL = https://api.github.com/repos/JuliaLang/Downloads.jl/tarball/$1 From d79478001f9ce8cad834c930f1600d53db199e70 Mon Sep 17 00:00:00 2001 From: Ajinkya Kokandakar Date: Fri, 21 Feb 2025 21:11:38 -0600 Subject: [PATCH 17/25] fixed clang url in build.md (#57501) Sorry this was supposed to be in an earlier PR but I missed this one. --- doc/src/devdocs/build/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/devdocs/build/build.md b/doc/src/devdocs/build/build.md index cb7a5bcfbed4bf..ff7e64926f1f7a 100644 --- a/doc/src/devdocs/build/build.md +++ b/doc/src/devdocs/build/build.md @@ -162,7 +162,7 @@ Notes for various architectures: Building Julia requires that the following software be installed: - **[GNU make](https://www.gnu.org/software/make)** — building dependencies. -- **[gcc & g++](https://gcc.gnu.org)** (>= 7.1) or **[Clang][clang]** (>= 5.0, >= 9.3 for Apple Clang) — compiling and linking C, C++. +- **[gcc & g++](https://gcc.gnu.org)** (>= 7.1) or **[Clang](https://clang.llvm.org)** (>= 5.0, >= 9.3 for Apple Clang) — compiling and linking C, C++. - **[libatomic](https://gcc.gnu.org)** — provided by **[gcc]** and needed to support atomic operations. - **[python](https://www.python.org/)** (>=2.7) — needed to build LLVM. - **[gfortran](https://gcc.gnu.org/fortran/)** — compiling and linking Fortran libraries. From 88aa27f482d19ae7ac0eb080c115640a0754d819 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 22 Feb 2025 05:59:25 -0500 Subject: [PATCH 18/25] strip Module filename from metadata (#57499) Fix #57358 --- src/staticdata.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 6cb157ad299fc8..ce7bd550cb65f6 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -794,6 +794,8 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ { jl_queue_for_serialization(s, m->name); jl_queue_for_serialization(s, m->parent); + if (jl_options.strip_metadata) + jl_queue_for_serialization(s, m->file); if (jl_options.trim) { jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); } else { @@ -1339,7 +1341,9 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t arraylist_push(&s->relocs_list, (void*)backref_id(s, jl_atomic_load_relaxed(&m->bindingkeyset), s->link_ids_relocs)); newm->file = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, file))); - arraylist_push(&s->relocs_list, (void*)backref_id(s, m->file, s->link_ids_relocs)); + arraylist_push(&s->relocs_list, (void*)backref_id(s, jl_options.strip_metadata ? jl_empty_sym : m->file , s->link_ids_relocs)); + if (jl_options.strip_metadata) + newm->line = 0; newm->usings_backedges = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings_backedges))); arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings_backedges, s->link_ids_relocs)); @@ -2593,7 +2597,7 @@ uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED { - jl_svec_t * bindings = jl_atomic_load_relaxed(&m->bindings); + jl_svec_t *bindings = jl_atomic_load_relaxed(&m->bindings); size_t l = jl_svec_len(bindings), i; arraylist_t bindings_list; arraylist_new(&bindings_list, 0); From 7b7ba33021ba1b427ee0f482999674536e7cbcef Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Sun, 23 Feb 2025 16:17:53 +0100 Subject: [PATCH 19/25] `Base.summarysize` for `Memory` with `Union` (#57508) Fixes the double accounting of the union byte array in `Base.summarysize` as described in #57506. If this is the correct fix, can it be backported to 1.11? Fix #57506 --- base/summarysize.jl | 7 +------ test/misc.jl | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/summarysize.jl b/base/summarysize.jl index 4f2646c7641b73..62b0ad08497782 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -149,13 +149,8 @@ function (ss::SummarySize)(obj::GenericMemory) datakey = unsafe_convert(Ptr{Cvoid}, obj) if !haskey(ss.seen, datakey) ss.seen[datakey] = true - dsize = sizeof(obj) + size += sizeof(obj) T = eltype(obj) - if isbitsunion(T) - # add 1 union selector byte for each element - dsize += length(obj) - end - size += dsize if !isempty(obj) && T !== Symbol && (!Base.allocatedinline(T) || (T isa DataType && !Base.datatype_pointerfree(T))) push!(ss.frontier_x, obj) push!(ss.frontier_i, 1) diff --git a/test/misc.jl b/test/misc.jl index bcc7ff69339a97..6c8d76fa1cd1a9 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -619,6 +619,11 @@ let z = Z53061[Z53061(S53061(rand(), (rand(),rand())), 0) for _ in 1:10^4] @test abs(summarysize(z) - 640000)/640000 <= 0.01 broken = Sys.WORD_SIZE == 32 && Sys.islinux() end +# issue #57506 +let len = 100, m1 = Memory{UInt8}(1:len), m2 = Memory{Union{Nothing,UInt8}}(1:len) + @test summarysize(m2) == summarysize(m1) + len +end + ## test conversion from UTF-8 to UTF-16 (for Windows APIs) # empty arrays From 9294132415cfdf2198b64d03f0fd58f8a4d3a258 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 23 Feb 2025 20:21:52 +0100 Subject: [PATCH 20/25] reword eachslice docs to allow other slices objects (#57428) The wording of `eachslice` was overly strict in enforcing a `Slices` return type, instead of allowing other objects with similar behavior but e.g. potentially additional behaviors. The existence of `AbstractSlices` suggests this was actually intended at some stage but did not make it into the docs. AxisKeys.jl already returns something else and it seems not to cause any problems, but it would be good to be able to do this without breaking the documented contract of the method. --------- Co-authored-by: Lilith Orion Hafner --- base/slicearray.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/slicearray.jl b/base/slicearray.jl index 215cf13f9651ea..7318181c1e8267 100644 --- a/base/slicearray.jl +++ b/base/slicearray.jl @@ -77,13 +77,13 @@ end """ eachslice(A::AbstractArray; dims, drop=true) -Create a [`Slices`](@ref) object that is an array of slices over dimensions `dims` of `A`, returning -views that select all the data from the other dimensions in `A`. `dims` can either be an -integer or a tuple of integers. +Create a sliced object, usually [`Slices`](@ref), that is an array of slices over dimensions +`dims` of `A`, returning views that select all the data from the other dimensions in `A`. +`dims` can either be an integer or a tuple of integers. -If `drop = true` (the default), the outer `Slices` will drop the inner dimensions, and +If `drop = true` (the default), the outer slices will drop the inner dimensions, and the ordering of the dimensions will match those in `dims`. If `drop = false`, then the -`Slices` will have the same dimensionality as the underlying array, with inner +slices object will have the same dimensionality as the underlying array, with inner dimensions having size 1. See [`stack`](@ref)`(slices; dims)` for the inverse of `eachslice(A; dims::Integer)`. From c284dfd80d7395da4a6332bf926298542c9028cd Mon Sep 17 00:00:00 2001 From: gitboy16 <82724369+gitboy16@users.noreply.github.com> Date: Mon, 24 Feb 2025 12:57:24 +0000 Subject: [PATCH 21/25] Update LICENSE.md year from 2024 to 2025 (#57516) --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index da8b6920491cc1..b7b53659b27728 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2009-2024: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors +Copyright (c) 2009-2025: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From 7b4c6d663886518d26d600a05d55718e8393748e Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Mon, 24 Feb 2025 08:01:54 -0500 Subject: [PATCH 22/25] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Sp?= =?UTF-8?q?arseArrays=20stdlib=20from=2072c7cac=20to=20ae727fe=20(#57512)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: SparseArrays URL: https://github.com/JuliaSparse/SparseArrays.jl.git Stdlib branch: main Julia branch: master Old commit: 72c7cac New commit: ae727fe Julia version: 1.13.0-DEV SparseArrays version: 1.12.0(Does not match) Bump invoked by: @ViralBShah Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaSparse/SparseArrays.jl/compare/72c7cac6bbf21367a3c2fbc5c50e908aea5984bb...ae727fe4a4881b995ee4de2c9664272dd0cb2705 ``` $ git log --oneline 72c7cac..ae727fe ae727fe Explicitly import `Matrix` and `Vector` in CHOLMOD (#601) ce852af cleanup support for Float32 and ComplexF32 (#597) ``` Co-authored-by: ViralBShah <744411+ViralBShah@users.noreply.github.com> --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 create mode 100644 deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 deleted file mode 100644 index 12c4f2ff976975..00000000000000 --- a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3f25f8a47a7945b55c9cc53ef489a55f diff --git a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 b/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 deleted file mode 100644 index 5daf7514ff4ed5..00000000000000 --- a/deps/checksums/SparseArrays-72c7cac6bbf21367a3c2fbc5c50e908aea5984bb.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -5fd827602430e79846d974661b039902a5ab6495f94af8292a3d66c3c3a07a0c59858bfa5bfa941bf8bf6418af98d1dd41b88aad4cc7c7355a8e56cad7f1f3ac diff --git a/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/md5 b/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/md5 new file mode 100644 index 00000000000000..9dd17d8a48632e --- /dev/null +++ b/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/md5 @@ -0,0 +1 @@ +26e4ff18b3ea1be9a17f0d93b57f678a diff --git a/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/sha512 b/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/sha512 new file mode 100644 index 00000000000000..f7b71a23ae9a3e --- /dev/null +++ b/deps/checksums/SparseArrays-ae727fe4a4881b995ee4de2c9664272dd0cb2705.tar.gz/sha512 @@ -0,0 +1 @@ +411f3aaa35ae9edcd75e0e6fb9306635ee02722bdf2803cfaec4f280212a41cac683e08dcc2ca8679d0f61037057e7f4365216b82f03c7b412311f98620d78d5 diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index f690a34eb5ae41..c5c819a7112edb 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = 72c7cac6bbf21367a3c2fbc5c50e908aea5984bb +SPARSEARRAYS_SHA1 = ae727fe4a4881b995ee4de2c9664272dd0cb2705 SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From 095fc99b69459d43c2ab3267ade9f491f52b92cc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 24 Feb 2025 08:57:57 -0500 Subject: [PATCH 23/25] make compile go faster with native caches (#57500) Not sure why we make sure compile go slow, so make it go fast. --- src/staticdata.c | 4 +--- sysimage.mk | 8 +++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index ce7bd550cb65f6..10057b69b8e6c2 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3654,9 +3654,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl htable_new(&new_dt_objs, 0); arraylist_new(&deser_sym, 0); - // in --build mode only use sysimg data, not precompiled native code - int imaging_mode = jl_generating_output() && !jl_options.incremental; - if (imaging_mode || jl_options.use_sysimage_native_code != JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES || IMAGE_NATIVE_CODE_TAINTED) { + if (jl_options.use_sysimage_native_code != JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES || IMAGE_NATIVE_CODE_TAINTED) { memset(&image->fptrs, 0, sizeof(image->fptrs)); image->gvars_base = NULL; IMAGE_NATIVE_CODE_TAINTED = 1; diff --git a/sysimage.mk b/sysimage.mk index ae6ce8699f417d..ddfe8029e19f0e 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -71,7 +71,13 @@ $(build_private_libdir)/basecompiler.ji: $(COMPILER_SRCS) --startup-file=no --warn-overwrite=yes -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) @mv $@.tmp $@ -$(build_private_libdir)/sys.ji: $(build_private_libdir)/basecompiler.ji $(JULIAHOME)/VERSION $(BASE_SRCS) $(STDLIB_SRCS) +$(build_private_libdir)/basecompiler-o.a $(build_private_libdir)/basecompiler-bc.a: $(build_private_libdir)/basecompiler-%.a : $(COMPILER_SRCS) + @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ + JULIA_NUM_THREADS=1 $(call spawn,$(JULIA_EXECUTABLE)) -C "$(JULIA_CPU_TARGET)" $(HEAPLIM) --output-$* $(call cygpath_w,$@).tmp \ + --startup-file=no --warn-overwrite=yes -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) + @mv $@.tmp $@ + +$(build_private_libdir)/sys.ji: $(build_private_libdir)/basecompiler.$(SHLIB_EXT) $(JULIAHOME)/VERSION $(BASE_SRCS) $(STDLIB_SRCS) @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ if ! JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ JULIA_NUM_THREADS=1 $(call spawn, $(JULIA_EXECUTABLE)) -g1 -O1 -C "$(JULIA_CPU_TARGET)" $(HEAPLIM) --output-ji $(call cygpath_w,$@).tmp $(JULIA_SYSIMG_BUILD_FLAGS) \ From 925a8fe44f5210c23831df06a19358614595546a Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 25 Feb 2025 06:41:44 +1100 Subject: [PATCH 24/25] Refactoring code that is specific to stock GC write barriers (#57237) This PR moves some code responsible for implementing write barriers around: the write barrier functions are lifted from `julia.h` into the new `gc-wb-*.h` files. --- src/Makefile | 4 +- src/gc-wb-mmtk.h | 84 +++++++++++++++++++++++++++++++++++++ src/gc-wb-stock.h | 102 +++++++++++++++++++++++++++++++++++++++++++++ src/julia.h | 103 ++++++---------------------------------------- 4 files changed, 201 insertions(+), 92 deletions(-) create mode 100644 src/gc-wb-mmtk.h create mode 100644 src/gc-wb-stock.h diff --git a/src/Makefile b/src/Makefile index 5c5e19217bd75e..5b5bc3b0601122 100644 --- a/src/Makefile +++ b/src/Makefile @@ -123,9 +123,9 @@ UV_HEADERS += uv/*.h endif PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,work-stealing-queue.h gc-interface.h gc-tls-common.h julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h jloptions.h) ifneq (${MMTK_PLAN},None) - PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h) + PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h gc-wb-mmtk.h) else - PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h) + PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h gc-wb-stock.h) endif ifeq ($(OS),WINNT) PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,win32_ucontext.h) diff --git a/src/gc-wb-mmtk.h b/src/gc-wb-mmtk.h new file mode 100644 index 00000000000000..c8c961544fabc8 --- /dev/null +++ b/src/gc-wb-mmtk.h @@ -0,0 +1,84 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +// ========================================================================= // +// Runtime Write-Barriers +// ========================================================================= // + +#ifndef JL_GC_WB_H +#define JL_GC_WB_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr); +extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr); +extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; + +#define MMTK_OBJECT_BARRIER (1) +// Stickyimmix needs write barrier. Immix does not need write barrier. +#ifdef MMTK_PLAN_IMMIX +#define MMTK_NEEDS_WRITE_BARRIER (0) +#endif +#ifdef MMTK_PLAN_STICKYIMMIX +#define MMTK_NEEDS_WRITE_BARRIER (1) +#endif + +// Directly call into MMTk for write barrier (debugging only) +STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT +{ + jl_task_t *ct = jl_current_task; + jl_ptls_t ptls = ct->ptls; + mmtk_object_reference_write_post(&ptls->gc_tls.mmtk_mutator, parent, ptr); +} + +// Inlined fastpath +STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSAFEPOINT +{ + if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { + intptr_t addr = (intptr_t) (void*) parent; + uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + jl_task_t *ct = jl_current_task; + jl_ptls_t ptls = ct->ptls; + mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, ptr); + } + } +} + +STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(parent, ptr); +} + +STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* +{ + mmtk_gc_wb_fast(ptr, (void*)0); +} + +STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(parent, (void*)0); +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, + jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(dest_owner, (void*)0); +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, + size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(owner, (void*)0); +} + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/gc-wb-stock.h b/src/gc-wb-stock.h new file mode 100644 index 00000000000000..4f27c74ef11109 --- /dev/null +++ b/src/gc-wb-stock.h @@ -0,0 +1,102 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +// ========================================================================= // +// Runtime Write-Barriers +// ========================================================================= // + +#ifndef JL_GC_WB_H +#define JL_GC_WB_H + +#ifdef __cplusplus +extern "C" { +#endif + +STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT +{ + // parent and ptr isa jl_value_t* + if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset + (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young + jl_gc_queue_root((jl_value_t*)parent); +} + +STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* +{ + // if ptr is old + if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { + jl_gc_queue_root((jl_value_t*)ptr); + } +} + +STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT +{ + // 3 == GC_OLD_MARKED + // ptr is an immutable object + if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) + return; // parent is young or in remset + if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) + return; // ptr is old and not in remset (thus it does not point to young) + jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); + const jl_datatype_layout_t *ly = dt->layout; + if (ly->npointers) + jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, + jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t done = 0; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + if (dest_p < src_p || dest_p > src_p + (*n)) { + for (; done < (*n); done++) { // copy forwards + void *val = jl_atomic_load_relaxed(src_p + done); + jl_atomic_store_release(dest_p + done, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + src_p += done; + dest_p += done; + } + else { + for (; done < (*n); done++) { // copy backwards + void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); + jl_atomic_store_release(dest_p + (*n) - done - 1, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + } + (*n) -= done; + } + } +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, + size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t elsz = dt->layout->size; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + dt = (jl_datatype_t*)jl_tparam1(dt); + for (size_t done = 0; done < n; done++) { // copy forwards + char* s = (char*)src_p+done*elsz; + if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) + jl_gc_queue_multiroot(owner, s, dt); + } + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/julia.h b/src/julia.h index 49c43c40c7b44d..0f37bbdf8e43e2 100644 --- a/src/julia.h +++ b/src/julia.h @@ -66,6 +66,7 @@ typedef struct _jl_taggedvalue_t jl_taggedvalue_t; typedef struct _jl_tls_states_t *jl_ptls_t; +typedef struct _jl_genericmemory_t jl_genericmemory_t; #ifdef JL_LIBRARY_EXPORTS #include "uv.h" @@ -1244,6 +1245,18 @@ STATIC_INLINE jl_value_t *jl_svecset( #define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length) #define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a)) +JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; +#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) + +STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; + +// write barriers +#ifndef MMTK_GC +#include "gc-wb-stock.h" +#else +#include "gc-wb-mmtk.h" +#endif + /* how - allocation style 0 = data is inlined @@ -1300,94 +1313,6 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set( } #endif -// GC write barriers - -STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT -{ - // parent and ptr isa jl_value_t* - if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset - (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young - jl_gc_queue_root((jl_value_t*)parent); -} - -STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* -{ - // if ptr is old - if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { - jl_gc_queue_root((jl_value_t*)ptr); - } -} - -STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT -{ - // 3 == GC_OLD_MARKED - // ptr is an immutable object - if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) - return; // parent is young or in remset - if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) - return; // ptr is old and not in remset (thus it does not point to young) - jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); - const jl_datatype_layout_t *ly = dt->layout; - if (ly->npointers) - jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); -} - -STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; - -STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, - jl_genericmemory_t *src, _Atomic(void*) * src_p, - size_t* n) JL_NOTSAFEPOINT -{ - if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - size_t done = 0; - if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { - if (dest_p < src_p || dest_p > src_p + (*n)) { - for (; done < (*n); done++) { // copy forwards - void *val = jl_atomic_load_relaxed(src_p + done); - jl_atomic_store_release(dest_p + done, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { - jl_gc_queue_root(dest_owner); - break; - } - } - src_p += done; - dest_p += done; - } - else { - for (; done < (*n); done++) { // copy backwards - void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); - jl_atomic_store_release(dest_p + (*n) - done - 1, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { - jl_gc_queue_root(dest_owner); - break; - } - } - } - (*n) -= done; - } - } -} - -STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, - size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT -{ - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - size_t elsz = dt->layout->size; - if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { - dt = (jl_datatype_t*)jl_tparam1(dt); - for (size_t done = 0; done < n; done++) { // copy forwards - char* s = (char*)src_p+done*elsz; - if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) - jl_gc_queue_multiroot(owner, s, dt); - } - } - } -} - STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT { jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; @@ -2402,8 +2327,6 @@ JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e JL_MAYBE_UNROOTED); JL_DLLEXPORT void JL_NORETURN jl_rethrow(void); JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED); JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e, jl_task_t *ct); -JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; -#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) extern JL_DLLIMPORT int jl_task_gcstack_offset; extern JL_DLLIMPORT int jl_task_ptls_offset; From f7b986d138b062795048e5992aea31aad5c8a306 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 24 Feb 2025 11:43:22 -0800 Subject: [PATCH 25/25] Fix typo in printed message and clarify a printed hint (#57511) The block of messages including a warning, two notes, and two hints regarding extending type constructors without qualification contains one typo. The last hint can also be clarified a bit, as it's currently ambiguous as to where the qualification needs to occur; it only needs to be qualified in the method definition. There may be a better way to clarify the qualification bit than what I have here, happy to amend further. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 848ced7058af11..0248ce011422f6 100644 --- a/src/module.c +++ b/src/module.c @@ -748,10 +748,10 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ else if (!b->did_print_implicit_import_admonition) { b->did_print_implicit_import_admonition = 1; jl_printf(JL_STDERR, "WARNING: Constructor for type \"%s\" was extended in `%s` without explicit qualification or import.\n" - " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.`\n" + " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.\n" " NOTE: This behavior may have differed in Julia versions prior to 1.12.\n" " Hint: If you intended to create a new generic function of the same name, use `function %s end`.\n" - " Hint: To silence the warning, qualify `%s` as `%s.%s` or explicitly `import %s: %s`\n", + " Hint: To silence the warning, qualify `%s` as `%s.%s` in the method signature or explicitly `import %s: %s`.\n", jl_symbol_name(var), jl_module_debug_name(m), jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var), jl_symbol_name(var), jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var),