From 811cb79244a1b8e2cf95e9451cfbb08f67795591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 17 Jan 2025 10:06:52 +0100 Subject: [PATCH 01/51] Add specs for `File.match?` (#15348) --- spec/std/file_spec.cr | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 0f88b2028c2f..fe572e710084 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1698,6 +1698,11 @@ describe "File" do assert_file_matches "a*", "abc" assert_file_matches "a*/b", "abc/b" assert_file_matches "*x", "xxx" + assert_file_matches "*.x", "a.x" + assert_file_matches "a/b/*.x", "a/b/c.x" + refute_file_matches "*.x", "a/b/c.x" + refute_file_matches "c.x", "a/b/c.x" + refute_file_matches "b/*.x", "a/b/c.x" end it "matches multiple expansions" do @@ -1719,6 +1724,21 @@ describe "File" do refute_file_matches "a*b*c*d*e*/f", "axbxcxdxexxx/fff" end + it "**" do + assert_file_matches "a/b/**", "a/b/c.x" + assert_file_matches "a/**", "a/b/c.x" + assert_file_matches "a/**/d.x", "a/b/c/d.x" + refute_file_matches "a/**b/d.x", "a/bb/c/d.x" + refute_file_matches "a/b**/*", "a/bb/c/d.x" + end + + it "** bugs (#15319)" do + refute_file_matches "a/**/*", "a/b/c/d.x" + assert_file_matches "a/b**/d.x", "a/bb/c/d.x" + refute_file_matches "**/*.x", "a/b/c.x" + assert_file_matches "**.x", "a/b/c.x" + end + it "** matches path separator" do assert_file_matches "a**", "ab/c" assert_file_matches "a**/b", "a/c/b" From cfd8ea1808c0c2b2942d6291ddf944c267c3a0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 17 Jan 2025 10:07:31 +0100 Subject: [PATCH 02/51] [CI] Add build shards to `mingw-w64` workflow (#15344) Co-authored-by: Julien Portalier --- .github/workflows/mingw-w64.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index f06efdd80161..3683351c71ca 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -60,6 +60,7 @@ jobs: mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm mingw-w64-ucrt-x86_64-libffi + mingw-w64-ucrt-x86_64-libyaml - name: Disable CRLF line ending substitution run: | @@ -87,6 +88,23 @@ jobs: run: | make install install_dlls deref_symlinks=1 PREFIX="$(pwd)/crystal" + - name: Download shards release + uses: actions/checkout@v4 + with: + repository: crystal-lang/shards + ref: v0.19.0 + path: shards + + - name: Build shards release + shell: msys2 {0} + working-directory: ./shards + run: make CRYSTAL=$(pwd)/../crystal/bin/crystal SHARDS=false release=1 + + - name: Package Shards + shell: msys2 {0} + run: | + make install PREFIX="$(pwd)/../crystal" + - name: Upload Crystal executable uses: actions/upload-artifact@v4 with: From 0fbdcc90f1e907995671a4387fcc22f957321d61 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 17 Jan 2025 10:09:59 +0100 Subject: [PATCH 03/51] Fix GC `sig_suspend`, `sig_resume` for `gc_none` (#15349) The gc_none interface doesn't define the `sig_suspend` nor `sig_resume` class methods. The program should still compile but commit 57017f6 improperly checks for the method existence, and the methods are always required and compilation fails. --- src/crystal/system/unix/pthread.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 73aa2a652ca1..98629a70fbb6 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -269,16 +269,16 @@ module Crystal::System::Thread {% end %} def self.sig_suspend : ::Signal - if GC.responds_to?(:sig_suspend) - GC.sig_suspend + if (gc = GC).responds_to?(:sig_suspend) + gc.sig_suspend else ::Signal.new(SIG_SUSPEND) end end def self.sig_resume : ::Signal - if GC.responds_to?(:sig_resume) - GC.sig_resume + if (gc = GC).responds_to?(:sig_resume) + gc.sig_resume else ::Signal.new(SIG_RESUME) end From 2b544f19eb687faa66e210127bb7c8fb75e65167 Mon Sep 17 00:00:00 2001 From: "Billy.Zheng" Date: Fri, 17 Jan 2025 17:10:59 +0800 Subject: [PATCH 04/51] Fix: Process::Status#exit_code doc (#15351) --- src/process/status.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/process/status.cr b/src/process/status.cr index 268d2f9e52d6..78cff49f0dc9 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -245,7 +245,7 @@ class Process::Status # Raises `RuntimeError` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code # => 1 + # Process.run("true").exit_code # => 0 # Process.run("exit 123", shell: true).exit_code # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code # ``` @@ -258,7 +258,7 @@ class Process::Status # Returns `nil` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code? # => 1 + # Process.run("true").exit_code? # => 0 # Process.run("exit 123", shell: true).exit_code? # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil # ``` From c4682db98e482a084bf03261614d319ea55b975e Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 18 Jan 2025 12:40:41 +0100 Subject: [PATCH 05/51] Fix: Hash `@indices` can grow larger than Int32::MAX bytes (#15347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Hash` can theoretically hold up to `Int32::MAX` entries, but `@indices` grows faster. Using `Int32` maths to calculate the memory size for `@indices` limits capacity to `Int32::MAX // 4`. This patch uses `Int64` maths instead to enable bigger hash sizes. Co-authored-by: Johannes Müller --- spec/manual/hash_large_spec.cr | 8 ++++++++ src/hash.cr | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 spec/manual/hash_large_spec.cr diff --git a/spec/manual/hash_large_spec.cr b/spec/manual/hash_large_spec.cr new file mode 100644 index 000000000000..d4ca4af96a8f --- /dev/null +++ b/spec/manual/hash_large_spec.cr @@ -0,0 +1,8 @@ +require "spec" + +it "creates Hash at maximum capacity" do + # we don't try to go as high as Int32::MAX because it would allocate 18GB of + # memory in total. This already tests for Int32 overflows while 'only' needing + # 4.5GB of memory. + Hash(Int32, Int32).new(initial_capacity: (Int32::MAX // 4) + 1) +end diff --git a/src/hash.cr b/src/hash.cr index 9b2936ddd618..1be6543d730c 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -236,12 +236,14 @@ class Hash(K, V) # Translate initial capacity to the nearest power of 2, but keep it a minimum of 8. if initial_capacity < 8 initial_entries_size = 8 + elsif initial_capacity > 2**30 + initial_entries_size = Int32::MAX else initial_entries_size = Math.pw2ceil(initial_capacity) end # Because we always keep indice_size >= entries_size * 2 - initial_indices_size = initial_entries_size * 2 + initial_indices_size = initial_entries_size.to_u64 * 2 @entries = malloc_entries(initial_entries_size) @@ -830,7 +832,7 @@ class Hash(K, V) # The actual number of bytes needed to allocate `@indices`. private def indices_malloc_size(size) - size * @indices_bytesize + size.to_u64 * @indices_bytesize end # Reallocates `size` number of indices for `@indices`. From 39aaae569f22e39a155dcfe5a962bd2660cc828f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 20 Jan 2025 11:56:31 +0100 Subject: [PATCH 06/51] Query runtime version of LLVM (#15355) Use the runtime version of libllvm (via `LLVMGetVersion`, if available) for `crystal --version` and `Crystal::LLVM_VERSION` constant. The version of a dynamically loaded library might differ from `LibLLVM::VERSION` which is the version at build time. --- src/compiler/crystal/config.cr | 7 ++++++- src/llvm/lib_llvm/core.cr | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index 2f71aa49815c..da85583de3aa 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -11,7 +11,12 @@ module Crystal end def self.llvm_version - LibLLVM::VERSION + {% if LibLLVM.has_method?(:get_version) %} + LibLLVM.get_version(out major, out minor, out patch) + "#{major}.#{minor}.#{patch}" + {% else %} + LibLLVM::VERSION + {% end %} end def self.description diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index 7137501fdb31..ef7b8f10b567 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -17,6 +17,10 @@ lib LibLLVM fun dispose_message = LLVMDisposeMessage(message : Char*) + {% unless LibLLVM::IS_LT_160 %} + fun get_version = LLVMGetVersion(major : UInt*, minor : UInt*, patch : UInt*) : Void + {% end %} + fun create_context = LLVMContextCreate : ContextRef fun dispose_context = LLVMContextDispose(c : ContextRef) From 8d02c8b82d3446a57382f60d28dc3c916dd44b2a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 20 Jan 2025 22:22:23 +0100 Subject: [PATCH 07/51] Rework initialization of constants & class variables (#15333) Changes the `flag` that keeps track of initialization status of constants and class variables to have 3 states instead of 2. The third one indicates that the value is currently being initialized and allows detecting recursion. Previously we used an array to keep track of values being currently initialized. This array is now unnecessary. The signature of `__crystal_once` changes: it now takes an Int8 (i8) instead of Bool (i1) and drops the once `state` pointer which isn't needed anymore. So `__crystal_once_init` no longer initializes a state pointer and returns nil. Also introduces a fast path for the (very likely) scenario that the variable is already initialized which doesn't need a mutex. Also introduces an LLVM optimization that instructs LLVM to optimize away repeated calls to `__crystal_once` for the same initializer. Requires a new compiler build to benefit from the improvement. The legacy versions of `__crystal_once` and `__crystal_once_init` are still supported by both the stdlib and the compiler to keep both forward & backward compatibility (1.15 and below can build 1.16+ and 1.16+ can build 1.15 and below). A follow-up could leverage `ReferenceStorage` and `.unsafe_construct` to inline the `Mutex` instead of allocating in the GC heap. Along with #15330 then `__crystal_once_init` could become allocation free, which could prove useful for such a core/low level feature. Co-authored-by: David Keller --- src/compiler/crystal/codegen/class_var.cr | 6 +- src/compiler/crystal/codegen/const.cr | 4 +- src/compiler/crystal/codegen/once.cr | 54 ++++--- src/crystal/once.cr | 172 ++++++++++++++++------ src/intrinsics.cr | 17 +++ 5 files changed, 188 insertions(+), 65 deletions(-) diff --git a/src/compiler/crystal/codegen/class_var.cr b/src/compiler/crystal/codegen/class_var.cr index 07d8ed0f96b1..a31a9b00bac5 100644 --- a/src/compiler/crystal/codegen/class_var.cr +++ b/src/compiler/crystal/codegen/class_var.cr @@ -25,8 +25,8 @@ class Crystal::CodeGenVisitor initialized_flag_name = class_var_global_initialized_name(class_var) initialized_flag = @main_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @main_mod.globals.add(@main_llvm_context.int1, initialized_flag_name) - initialized_flag.initializer = @main_llvm_context.int1.const_int(0) + initialized_flag = @main_mod.globals.add(@main_llvm_context.int8, initialized_flag_name) + initialized_flag.initializer = @main_llvm_context.int8.const_int(0) initialized_flag.linkage = LLVM::Linkage::Internal if @single_module initialized_flag.thread_local = true if class_var.thread_local? end @@ -61,7 +61,7 @@ class Crystal::CodeGenVisitor initialized_flag_name = class_var_global_initialized_name(class_var) initialized_flag = @llvm_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @llvm_mod.globals.add(llvm_context.int1, initialized_flag_name) + initialized_flag = @llvm_mod.globals.add(llvm_context.int8, initialized_flag_name) initialized_flag.thread_local = true if class_var.thread_local? end end diff --git a/src/compiler/crystal/codegen/const.cr b/src/compiler/crystal/codegen/const.cr index ec306e97296d..decfb3945e20 100644 --- a/src/compiler/crystal/codegen/const.cr +++ b/src/compiler/crystal/codegen/const.cr @@ -64,8 +64,8 @@ class Crystal::CodeGenVisitor initialized_flag_name = const.initialized_llvm_name initialized_flag = @main_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @main_mod.globals.add(@main_llvm_context.int1, initialized_flag_name) - initialized_flag.initializer = @main_llvm_context.int1.const_int(0) + initialized_flag = @main_mod.globals.add(@main_llvm_context.int8, initialized_flag_name) + initialized_flag.initializer = @main_llvm_context.int8.const_int(0) initialized_flag.linkage = LLVM::Linkage::Internal if @single_module end initialized_flag diff --git a/src/compiler/crystal/codegen/once.cr b/src/compiler/crystal/codegen/once.cr index 2e91267c1f52..ba08f0e7958b 100644 --- a/src/compiler/crystal/codegen/once.cr +++ b/src/compiler/crystal/codegen/once.cr @@ -7,31 +7,49 @@ class Crystal::CodeGenVisitor if once_init_fun = typed_fun?(@main_mod, ONCE_INIT) once_init_fun = check_main_fun ONCE_INIT, once_init_fun - once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) - once_state_global.linkage = LLVM::Linkage::Internal if @single_module - once_state_global.initializer = once_init_fun.type.return_type.null - - state = call once_init_fun - store state, once_state_global + if once_init_fun.type.return_type.void? + call once_init_fun + else + # legacy (kept for backward compatibility): the compiler must save the + # state returned by __crystal_once_init + once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) + once_state_global.linkage = LLVM::Linkage::Internal if @single_module + once_state_global.initializer = once_init_fun.type.return_type.null + + state = call once_init_fun + store state, once_state_global + end end end def run_once(flag, func : LLVMTypedFunction) once_fun = main_fun(ONCE) - once_init_fun = main_fun(ONCE_INIT) - - # both of these should be Void* - once_state_type = once_init_fun.type.return_type - once_initializer_type = once_fun.func.params.last.type + once_fun_params = once_fun.func.params + once_initializer_type = once_fun_params.last.type # must be Void* + initializer = pointer_cast(func.func.to_value, once_initializer_type) - once_state_global = @llvm_mod.globals[ONCE_STATE]? || begin - global = @llvm_mod.globals.add(once_state_type, ONCE_STATE) - global.linkage = LLVM::Linkage::External - global + if once_fun_params.size == 2 + args = [flag, initializer] + else + # legacy (kept for backward compatibility): the compiler must pass the + # state returned by __crystal_once_init to __crystal_once as the first + # argument + once_init_fun = main_fun(ONCE_INIT) + once_state_type = once_init_fun.type.return_type # must be Void* + + once_state_global = @llvm_mod.globals[ONCE_STATE]? || begin + global = @llvm_mod.globals.add(once_state_type, ONCE_STATE) + global.linkage = LLVM::Linkage::External + global + end + + state = load(once_state_type, once_state_global) + {% if LibLLVM::IS_LT_150 %} + flag = bit_cast(flag, @llvm_context.int1.pointer) # cast Int8* to Bool* + {% end %} + args = [state, flag, initializer] end - state = load(once_state_type, once_state_global) - initializer = pointer_cast(func.func.to_value, once_initializer_type) - call once_fun, [state, flag, initializer] + call once_fun, args end end diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 56eea2be693a..7b6c7e6ed0f2 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -1,54 +1,142 @@ -# This file defines the functions `__crystal_once_init` and `__crystal_once` expected -# by the compiler. `__crystal_once` is called each time a constant or class variable -# has to be initialized and is its responsibility to verify the initializer is executed -# only once. `__crystal_once_init` is executed only once at the beginning of the program -# and the result is passed on each call to `__crystal_once`. - -# This implementation uses an array to store the initialization flag pointers for each value -# to find infinite loops and raise an error. In multithread mode a mutex is used to -# avoid race conditions between threads. - -# :nodoc: -class Crystal::OnceState - @rec = [] of Bool* - - def once(flag : Bool*, initializer : Void*) - unless flag.value - if @rec.includes?(flag) - raise "Recursion while initializing class variables and/or constants" +# This file defines two functions expected by the compiler: +# +# - `__crystal_once_init`: executed only once at the beginning of the program +# and, for the legacy implementation, the result is passed on each call to +# `__crystal_once`. +# +# - `__crystal_once`: called each time a constant or class variable has to be +# initialized and is its responsibility to verify the initializer is executed +# only once and to fail on recursion. + +# In multithread mode a mutex is used to avoid race conditions between threads. +# +# On Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new +# thread even without the `preview_mt` flag, and the thread can also reference +# Crystal constants, leading to race conditions, so we always enable the mutex. + +{% if compare_versions(Crystal::VERSION, "1.16.0-dev") >= 0 %} + # This implementation uses an enum over the initialization flag pointer for + # each value to find infinite loops and raise an error. + + module Crystal + # :nodoc: + enum OnceState : Int8 + Processing = -1 + Uninitialized = 0 + Initialized = 1 + end + + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex = uninitialized Mutex + + # :nodoc: + def self.once_mutex=(@@once_mutex : Mutex) end - @rec << flag + {% end %} - Proc(Nil).new(initializer, Pointer(Void).null).call - flag.value = true + # :nodoc: + # Using @[NoInline] so LLVM optimizes for the hot path (var already + # initialized). + @[NoInline] + def self.once(flag : OnceState*, initializer : Void*) : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex.synchronize { once_exec(flag, initializer) } + {% else %} + once_exec(flag, initializer) + {% end %} - @rec.pop + # safety check, and allows to safely call `Intrinsics.unreachable` in + # `__crystal_once` + unless flag.value.initialized? + System.print_error "BUG: failed to initialize constant or class variable\n" + LibC._exit(1) + end + end + + private def self.once_exec(flag : OnceState*, initializer : Void*) : Nil + case flag.value + in .initialized? + return + in .uninitialized? + flag.value = :processing + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = :initialized + in .processing? + raise "Recursion while initializing class variables and/or constants" + end end end - # on Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new - # thread even without the `preview_mt` flag, and the thread can also reference - # Crystal constants, leading to race conditions, so we always enable the mutex - # TODO: can this be improved? - {% if flag?(:preview_mt) || flag?(:win32) %} - @mutex = Mutex.new(:reentrant) + # :nodoc: + fun __crystal_once_init : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + Crystal.once_mutex = Mutex.new(:reentrant) + {% end %} + end + + # :nodoc: + # + # Using `@[AlwaysInline]` allows LLVM to optimize const accesses. Since this + # is a `fun` the function will still appear in the symbol table, though it + # will never be called. + @[AlwaysInline] + fun __crystal_once(flag : Crystal::OnceState*, initializer : Void*) : Nil + return if flag.value.initialized? + + Crystal.once(flag, initializer) + # tell LLVM that it can optimize away repeated `__crystal_once` calls for + # this global (e.g. repeated access to constant in a single funtion); + # this is truly unreachable otherwise `Crystal.once` would have panicked + Intrinsics.unreachable unless flag.value.initialized? + end +{% else %} + # This implementation uses a global array to store the initialization flag + # pointers for each value to find infinite loops and raise an error. + + # :nodoc: + class Crystal::OnceState + @rec = [] of Bool* + + @[NoInline] def once(flag : Bool*, initializer : Void*) unless flag.value - @mutex.synchronize do - previous_def + if @rec.includes?(flag) + raise "Recursion while initializing class variables and/or constants" end + @rec << flag + + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = true + + @rec.pop end end - {% end %} -end - -# :nodoc: -fun __crystal_once_init : Void* - Crystal::OnceState.new.as(Void*) -end - -# :nodoc: -fun __crystal_once(state : Void*, flag : Bool*, initializer : Void*) - state.as(Crystal::OnceState).once(flag, initializer) -end + + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + + @[NoInline] + def once(flag : Bool*, initializer : Void*) + unless flag.value + @mutex.synchronize do + previous_def + end + end + end + {% end %} + end + + # :nodoc: + fun __crystal_once_init : Void* + Crystal::OnceState.new.as(Void*) + end + + # :nodoc: + @[AlwaysInline] + fun __crystal_once(state : Void*, flag : Bool*, initializer : Void*) + return if flag.value + state.as(Crystal::OnceState).once(flag, initializer) + Intrinsics.unreachable unless flag.value + end +{% end %} diff --git a/src/intrinsics.cr b/src/intrinsics.cr index dc83ab91c884..3ccc47996e0b 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -354,6 +354,23 @@ module Intrinsics macro va_end(ap) ::LibIntrinsics.va_end({{ap}}) end + + # Should codegen to the following LLVM IR (before being inlined): + # ``` + # define internal void @"*Intrinsics::unreachable:NoReturn"() #12 { + # entry: + # unreachable + # } + # ``` + # + # Can be used like `@llvm.assume(i1 cond)` as `unreachable unless (assumption)`. + # + # WARNING: the behaviour of the program is undefined if the assumption is broken! + @[AlwaysInline] + def self.unreachable : NoReturn + x = uninitialized NoReturn + x + end end macro debugger From 5a245d905717ff9ea8a62512d8a5e0fefa721959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 21 Jan 2025 09:03:59 +0100 Subject: [PATCH 08/51] Fix: abstract EventLoop::Polling#system_add invalid signature The abstract method refers to the non existing `Index` type. Weirdly the compiler won't complain until someone defines an `Index` type. Resolves #15357 --- src/crystal/event_loop/polling.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 2fe86ad5b649..3eb17c0e313e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -515,7 +515,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop private abstract def system_run(blocking : Bool, & : Fiber ->) : Nil # Add *fd* to the polling system, setting *index* as user data. - protected abstract def system_add(fd : Int32, index : Index) : Nil + protected abstract def system_add(fd : Int32, index : Arena::Index) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. # From b103c28b717ea58079deb2557b598ecf0659fe19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 21 Jan 2025 12:19:09 +0100 Subject: [PATCH 09/51] Add `LLVM.version` (#15354) --- spec/std/llvm/llvm_spec.cr | 4 ++++ src/compiler/crystal/config.cr | 11 +---------- src/compiler/crystal/program.cr | 2 +- src/llvm.cr | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/spec/std/llvm/llvm_spec.cr b/spec/std/llvm/llvm_spec.cr index e39398879e5d..a863d070199a 100644 --- a/spec/std/llvm/llvm_spec.cr +++ b/spec/std/llvm/llvm_spec.cr @@ -2,6 +2,10 @@ require "spec" require "llvm" describe LLVM do + it ".version" do + LLVM.version.should eq LibLLVM::VERSION + end + describe ".normalize_triple" do it "works" do LLVM.normalize_triple("x86_64-apple-macos").should eq("x86_64-apple-macos") diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index da85583de3aa..021a1717d4b5 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -10,15 +10,6 @@ module Crystal {{ read_file("#{__DIR__}/../../VERSION").chomp }} end - def self.llvm_version - {% if LibLLVM.has_method?(:get_version) %} - LibLLVM.get_version(out major, out minor, out patch) - "#{major}.#{minor}.#{patch}" - {% else %} - LibLLVM::VERSION - {% end %} - end - def self.description String.build do |io| io << "Crystal " << version @@ -27,7 +18,7 @@ module Crystal io << "\n\nThe compiler was not built in release mode." unless release_mode? - io << "\n\nLLVM: " << llvm_version + io << "\n\nLLVM: " << LLVM.version io << "\nDefault target: " << host_target io << "\n" end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index bab4e22b9fba..840afd2b6552 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -316,7 +316,7 @@ module Crystal define_crystal_string_constant "VERSION", Crystal::Config.version, <<-MD The version of the Crystal compiler. MD - define_crystal_string_constant "LLVM_VERSION", Crystal::Config.llvm_version, <<-MD + define_crystal_string_constant "LLVM_VERSION", LLVM.version, <<-MD The version of LLVM used by the Crystal compiler. MD define_crystal_string_constant "HOST_TRIPLE", Crystal::Config.host_target.to_s, <<-MD diff --git a/src/llvm.cr b/src/llvm.cr index 84c9dc89aa8f..d431a9c4c4d0 100644 --- a/src/llvm.cr +++ b/src/llvm.cr @@ -4,6 +4,22 @@ require "c/string" module LLVM @@initialized = false + # Returns the runtime version of LLVM. + # + # Starting with LLVM 16, this method returns the version as reported by + # `LLVMGetVersion` at runtime. Older versions of LLVM do not expose this + # information, so the value falls back to `LibLLVM::VERSION` which is + # determined at compile time and might slightly be out of sync to the + # dynamic library loaded at runtime. + def self.version + {% if LibLLVM.has_method?(:get_version) %} + LibLLVM.get_version(out major, out minor, out patch) + "#{major}.#{minor}.#{patch}" + {% else %} + LibLLVM::VERSION + {% end %} + end + def self.init_x86 : Nil return if @@initialized_x86 @@initialized_x86 = true From 35ecb7d448b60f413850528d74ac0b5d0c18da1f Mon Sep 17 00:00:00 2001 From: Luis Lavena Date: Thu, 23 Jan 2025 14:55:19 +0100 Subject: [PATCH 10/51] Corrects lib lookup in case-sensitive OSes (#15362) Downcase name of Windows import libraries to avoid duplicates references (Eg. kernel32 and Kernel32) but to correctly find the libraries on case-sensitive filesystems, like when cross-compiling from Linux. While on Windows `Kernel32` and `kernel32` result in the correct library found for linking, it does not work the same way when cross-compiling. This also applies to Winsock2 (ws2_32) usage. The change normalizes on the usage of downcased names for these libraries (which will have no impact when compiling natively on Windows). Prior to this change: ```console $ crystal build --cross-compile --target x86_64-windows-gnu 1.cr cc 1.obj -o 1 -Wl,--stack,0x800000 -L/usr/local/bin/../lib/crystal -lgc -lpthread \ -ldl -municode -lntdll -liconv -ladvapi32 -lshell32 \ -lole32 -lWS2_32 -lkernel32 -lKernel32 ``` After: ```console $ crystal build --cross-compile --target x86_64-windows-gnu 1.cr cc 1.obj -o 1 -Wl,--stack,0x800000 -L/usr/local/bin/../lib/crystal -lgc -lpthread \ -ldl -municode -lntdll -liconv -ladvapi32 -lshell32 \ -lole32 -lws2_32 -lkernel32 ``` --- src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr index 5612233553d9..67b114bfc80f 100644 --- a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr @@ -1,6 +1,6 @@ require "c/winnt" -@[Link("Kernel32")] +@[Link("kernel32")] lib LibC alias FARPROC = Void* diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 21ae8baba852..9a2dde0ca146 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -3,7 +3,7 @@ require "./basetsd" require "./guiddef" require "./winbase" -@[Link("WS2_32")] +@[Link("ws2_32")] lib LibC alias SOCKET = UINT_PTR From df06679ac0ceda8741056d96495f7ef60bc15081 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 23 Jan 2025 14:58:21 +0100 Subject: [PATCH 11/51] Fix `Reference#exec_recursive`, `#exec_recursive_clone` to be fiber aware (#15361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recursion of the `#inspect`, `#pretty_print` and `#clone` methods is handled by a per-thread global hash. But if any of these methods yields —which will happen when writing to an IO— another fiber running on the same thread will falsely detect a recursion if it inspects the same object. This patch moves the backing hashes to instance variables of `Fiber`, using a per-fiber hash instead of per-thread/process hash. References #15088 --- src/fiber.cr | 18 ++++++++++++++++++ src/reference.cr | 49 ++++-------------------------------------------- 2 files changed, 22 insertions(+), 45 deletions(-) diff --git a/src/fiber.cr b/src/fiber.cr index b34a8762037d..b27c34fd6b36 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -162,6 +162,10 @@ class Fiber @timeout_event.try &.free @timeout_select_action = nil + # Additional cleanup (avoid stale references) + @exec_recursive_hash = nil + @exec_recursive_clone_hash = nil + @alive = false {% unless flag?(:interpreted) %} Crystal::Scheduler.stack_pool.release(@stack) @@ -331,4 +335,18 @@ class Fiber @current_thread.lazy_get end {% end %} + + # :nodoc: + # + # See `Reference#exec_recursive` for details. + def exec_recursive_hash + @exec_recursive_hash ||= Hash({UInt64, Symbol}, Nil).new + end + + # :nodoc: + # + # See `Reference#exec_recursive_clone` for details. + def exec_recursive_clone_hash + @exec_recursive_clone_hash ||= Hash(UInt64, UInt64).new + end end diff --git a/src/reference.cr b/src/reference.cr index f70697282fa0..42bdcba2327a 100644 --- a/src/reference.cr +++ b/src/reference.cr @@ -1,7 +1,3 @@ -{% if flag?(:preview_mt) %} - require "crystal/thread_local_value" -{% end %} - # `Reference` is the base class of classes you define in your program. # It is set as a class' superclass when you don't specify one: # @@ -180,28 +176,9 @@ class Reference io << '>' end - # :nodoc: - module ExecRecursive - # NOTE: can't use `Set` here because of prelude require order - alias Registry = Hash({UInt64, Symbol}, Nil) - - {% if flag?(:preview_mt) %} - @@exec_recursive = Crystal::ThreadLocalValue(Registry).new - {% else %} - @@exec_recursive = Registry.new - {% end %} - - def self.hash - {% if flag?(:preview_mt) %} - @@exec_recursive.get { Registry.new } - {% else %} - @@exec_recursive - {% end %} - end - end - private def exec_recursive(method, &) - hash = ExecRecursive.hash + # NOTE: can't use `Set` because of prelude require order + hash = Fiber.current.exec_recursive_hash key = {object_id, method} hash.put(key, nil) do yield @@ -211,25 +188,6 @@ class Reference false end - # :nodoc: - module ExecRecursiveClone - alias Registry = Hash(UInt64, UInt64) - - {% if flag?(:preview_mt) %} - @@exec_recursive = Crystal::ThreadLocalValue(Registry).new - {% else %} - @@exec_recursive = Registry.new - {% end %} - - def self.hash - {% if flag?(:preview_mt) %} - @@exec_recursive.get { Registry.new } - {% else %} - @@exec_recursive - {% end %} - end - end - # Helper method to perform clone by also checking recursiveness. # When clone is wanted, call this method. Then create the clone # instance without any contents (don't fill it out yet), then @@ -249,7 +207,8 @@ class Reference # end # ``` private def exec_recursive_clone(&) - hash = ExecRecursiveClone.hash + # NOTE: can't use `Set` because of prelude require order + hash = Fiber.current.exec_recursive_clone_hash clone_object_id = hash[object_id]? unless clone_object_id clone_object_id = yield(hash).object_id From bac59ecd5a33bae43300a3499f5723e30e57cfb7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 24 Jan 2025 10:42:08 +0100 Subject: [PATCH 12/51] Fix invalid returns in class getter's lazy evaluation blocks (#15364) Using `return` in a class getter will immediately return from the generated method, never setting the class variable to the returned value, which is kept nil and so the initializer code is called repeatedly instead of being called _once_ then cached. --- spec/std/socket/spec_helper.cr | 4 +++- spec/support/time.cr | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/std/socket/spec_helper.cr b/spec/std/socket/spec_helper.cr index 276e2f4195f2..38aadaf802d9 100644 --- a/spec/std/socket/spec_helper.cr +++ b/spec/std/socket/spec_helper.cr @@ -2,7 +2,9 @@ require "spec" require "socket" module SocketSpecHelper - class_getter?(supports_ipv6 : Bool) do + class_getter?(supports_ipv6 : Bool) { detect_supports_ipv6? } + + private def self.detect_supports_ipv6? : Bool TCPServer.open("::1", 0) { return true } false rescue Socket::Error diff --git a/spec/support/time.cr b/spec/support/time.cr index d550a83af2c3..2b4738b5d71e 100644 --- a/spec/support/time.cr +++ b/spec/support/time.cr @@ -72,7 +72,9 @@ end # Enable the `SeTimeZonePrivilege` privilege before changing the system time # zone. This is necessary because the privilege is by default granted but # disabled for any new process. This only needs to be done once per run. - class_getter? time_zone_privilege_enabled : Bool do + class_getter?(time_zone_privilege_enabled : Bool) { detect_time_zone_privilege_enabled? } + + private def self.detect_time_zone_privilege_enabled? : Bool if LibC.LookupPrivilegeValueW(nil, SeTimeZonePrivilege, out time_zone_luid) == 0 raise RuntimeError.from_winerror("LookupPrivilegeValueW") end From 1bc54ca22074248faf1832ed10ff7081044d8019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 25 Jan 2025 08:56:40 +0100 Subject: [PATCH 13/51] Fix `make uninstall` to remove fish completion (#15367) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6328415d4ed7..f45e3c52db8e 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,7 @@ uninstall: ## Uninstall the compiler from DESTDIR rm -f "$(DESTDIR)$(PREFIX)/share/bash-completion/completions/crystal" rm -f "$(DESTDIR)$(PREFIX)/share/zsh/site-functions/_crystal" + rm -f "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/crystal.fish" .PHONY: install_docs install_docs: docs ## Install docs at DESTDIR From 9e475f6442465b4dda485ff12d2dfd70cda25efe Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 25 Jan 2025 22:48:22 +0100 Subject: [PATCH 14/51] Refactor explicit init of class var for `Thread`, `Fiber` in `Crystal.init_runtime` (#15369) Adds `Crystal.init_runtime` which calls `Thread.init`, `Fiber.init` to explicitly initialize the class variables defined on `Thread` and `Fiber`. In order to implement #14905 we'll need the ability to access `Fiber.current` which depends on `Thread.current` but if these accessors use the class getter/property macros then we'll enter an infinite recursion. --- src/crystal/main.cr | 17 +++++++++++++++-- src/crystal/system/thread.cr | 13 ++++++++++++- src/crystal/system/unix/pthread.cr | 29 ++++++++++++++++++++--------- src/crystal/system/wasi/thread.cr | 14 +++++++++++++- src/crystal/system/win32/thread.cr | 27 +++++++++++++++++++-------- src/fiber.cr | 10 +++++++++- 6 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 9b4384f16a8c..c26199fcdb4a 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -8,7 +8,7 @@ end module Crystal # Defines the main routine run by normal Crystal programs: # - # - Initializes the GC + # - Initializes runtime requirements (GC, ...) # - Invokes the given *block* # - Handles unhandled exceptions # - Invokes `at_exit` handlers @@ -37,6 +37,8 @@ module Crystal {% if flag?(:tracing) %} Crystal::Tracing.init {% end %} GC.init + init_runtime + status = begin yield @@ -48,6 +50,14 @@ module Crystal exit(status, ex) end + # :nodoc: + def self.init_runtime : Nil + # `__crystal_once` directly or indirectly depends on `Fiber` and `Thread` + # so we explicitly initialize their class vars + Thread.init + Fiber.init + end + # :nodoc: def self.exit(status : Int32, exception : Exception?) : Int32 status = Crystal::AtExitHandlers.run status, exception @@ -130,7 +140,10 @@ fun main(argc : Int32, argv : UInt8**) : Int32 Crystal.main(argc, argv) end -{% if flag?(:win32) %} +{% if flag?(:interpreted) %} + # the interpreter doesn't call Crystal.main(&) + Crystal.init_runtime +{% elsif flag?(:win32) %} require "./system/win32/wmain" {% elsif flag?(:wasi) %} require "./system/wasi/main" diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 92136d1f3989..878a27e4c578 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -2,6 +2,8 @@ module Crystal::System::Thread # alias Handle + # def self.init : Nil + # def self.new_handle(thread_obj : ::Thread) : Handle # def self.current_handle : Handle @@ -48,7 +50,16 @@ class Thread include Crystal::System::Thread # all thread objects, so the GC can see them (it doesn't scan thread locals) - protected class_getter(threads) { Thread::LinkedList(Thread).new } + @@threads = uninitialized Thread::LinkedList(Thread) + + protected def self.threads : Thread::LinkedList(Thread) + @@threads + end + + def self.init : Nil + @@threads = Thread::LinkedList(Thread).new + Crystal::System::Thread.init + end @system_handle : Crystal::System::Thread::Handle @exception : Exception? diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 98629a70fbb6..e91990689084 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -26,6 +26,16 @@ module Crystal::System::Thread raise RuntimeError.from_os_error("pthread_create", Errno.new(ret)) unless ret == 0 end + def self.init : Nil + {% if flag?(:musl) %} + @@main_handle = current_handle + {% elsif flag?(:openbsd) || flag?(:android) %} + ret = LibC.pthread_key_create(out current_key, nil) + raise RuntimeError.from_os_error("pthread_key_create", Errno.new(ret)) unless ret == 0 + @@current_key = current_key + {% end %} + end + def self.thread_proc(data : Void*) : Void* th = data.as(::Thread) @@ -53,13 +63,7 @@ module Crystal::System::Thread # Android appears to support TLS to some degree, but executables fail with # an underaligned TLS segment, see https://github.com/crystal-lang/crystal/issues/13951 {% if flag?(:openbsd) || flag?(:android) %} - @@current_key : LibC::PthreadKeyT - - @@current_key = begin - ret = LibC.pthread_key_create(out current_key, nil) - raise RuntimeError.from_os_error("pthread_key_create", Errno.new(ret)) unless ret == 0 - current_key - end + @@current_key = uninitialized LibC::PthreadKeyT def self.current_thread : ::Thread if ptr = LibC.pthread_getspecific(@@current_key) @@ -84,11 +88,18 @@ module Crystal::System::Thread end {% else %} @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + @@current_thread : ::Thread? + + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end def self.current_thread? : ::Thread? @@current_thread end + + def self.current_thread=(@@current_thread : ::Thread) + end {% end %} def self.sleep(time : ::Time::Span) : Nil @@ -169,7 +180,7 @@ module Crystal::System::Thread end {% if flag?(:musl) %} - @@main_handle : Handle = current_handle + @@main_handle = uninitialized Handle def self.current_is_main? current_handle == @@main_handle diff --git a/src/crystal/system/wasi/thread.cr b/src/crystal/system/wasi/thread.cr index 1e8f6957d526..d103c7d9fc44 100644 --- a/src/crystal/system/wasi/thread.cr +++ b/src/crystal/system/wasi/thread.cr @@ -1,6 +1,9 @@ module Crystal::System::Thread alias Handle = Nil + def self.init : Nil + end + def self.new_handle(thread_obj : ::Thread) : Handle raise NotImplementedError.new("Crystal::System::Thread.new_handle") end @@ -13,7 +16,16 @@ module Crystal::System::Thread raise NotImplementedError.new("Crystal::System::Thread.yield_current") end - class_property current_thread : ::Thread { ::Thread.new } + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end + + def self.current_thread? : ::Thread? + @@current_thread + end + + def self.current_thread=(@@current_thread : ::Thread) + end def self.sleep(time : ::Time::Span) : Nil req = uninitialized LibC::Timespec diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 9cb60f01ced8..2ff7ca438d87 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -20,6 +20,16 @@ module Crystal::System::Thread ) end + def self.init : Nil + {% if flag?(:gnu) %} + current_key = LibC.TlsAlloc + if current_key == LibC::TLS_OUT_OF_INDEXES + Crystal::System.panic("TlsAlloc()", WinError.value) + end + @@current_key = current_key + {% end %} + end + def self.thread_proc(data : Void*) : LibC::UInt # ensure that even in the case of stack overflow there is enough reserved # stack space for recovery (for the main thread this is done in @@ -47,13 +57,7 @@ module Crystal::System::Thread # MinGW does not support TLS correctly {% if flag?(:gnu) %} - @@current_key : LibC::DWORD = begin - current_key = LibC.TlsAlloc - if current_key == LibC::TLS_OUT_OF_INDEXES - Crystal::System.panic("TlsAlloc()", WinError.value) - end - current_key - end + @@current_key = uninitialized LibC::DWORD def self.current_thread : ::Thread th = current_thread? @@ -82,11 +86,18 @@ module Crystal::System::Thread end {% else %} @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + @@current_thread : ::Thread? + + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end def self.current_thread? : ::Thread? @@current_thread end + + def self.current_thread=(@@current_thread : ::Thread) + end {% end %} def self.sleep(time : ::Time::Span) : Nil diff --git a/src/fiber.cr b/src/fiber.cr index b27c34fd6b36..3d2fe9a89797 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -44,8 +44,16 @@ end # notifications that IO is ready or a timeout reached. When a fiber can be woken, # the event loop enqueues it in the scheduler class Fiber + @@fibers = uninitialized Thread::LinkedList(Fiber) + + protected def self.fibers : Thread::LinkedList(Fiber) + @@fibers + end + # :nodoc: - protected class_getter(fibers) { Thread::LinkedList(Fiber).new } + def self.init : Nil + @@fibers = Thread::LinkedList(Fiber).new + end @context : Context @stack : Void* From 659cf2570d0c1a79486c8f1c54e41656eaca4e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 13:40:39 +0100 Subject: [PATCH 15/51] Update distribution-scripts (#15368) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27b2d8e5440c..c530d0f2187a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "2f56eecf233f52229b93eab4acd2bca51f0f8edf" + default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 2697a0fa0630a154de676c77841c194eb3c8046a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 13:48:10 +0100 Subject: [PATCH 16/51] Update shards 0.19.1 (#15366) --- .github/workflows/win_build_portable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 6ff57effb3d2..601f1ad2eaa9 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 with: repository: crystal-lang/shards - ref: v0.19.0 + ref: v0.19.1 path: shards - name: Build shards release From d7b6a14ac790200e8a0218f65af09722c17d933d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 15:07:25 +0100 Subject: [PATCH 17/51] [CI] Add workflow for backporting PRs to release branches (#15372) --- .github/workflows/backport.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/backport.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000000..97c2657fcdf5 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,33 @@ +# WARNING: +# When extending this action, be aware that $GITHUB_TOKEN allows write access to +# the GitHub repository. This means that it should not evaluate user input in a +# way that allows code injection. + +name: Backport + +on: + pull_request_target: + types: [closed, labeled] + +permissions: + contents: write # so it can comment + pull-requests: write # so it can create pull requests + +jobs: + backport: + name: Backport Pull Request + if: github.repository_owner == 'crystal-lang' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Create backport PR + uses: korthout/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 + with: + # Config README: https://github.com/korthout/backport-action#backport-action + copy_labels_pattern: '^(breaking-change|security|topic:.*|kind:.*|platform:.*)$' + copy_milestone: true + pull_description: |- + Automated backport of #${pull_number} to `${target_branch}`, triggered by a label. From cfed9a5d83a73d42caa8074d7b3b7f3f4a879838 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 09:45:51 +0100 Subject: [PATCH 18/51] Update cygwin/cygwin-install-action action to v5 (#15346) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/win.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index aacf1a4aae4f..4075d6968e14 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -24,7 +24,7 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up Cygwin - uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + uses: cygwin/cygwin-install-action@f61179d72284ceddc397ed07ddb444d82bf9e559 # v5 with: packages: make install-dir: C:\cygwin64 @@ -103,7 +103,7 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up Cygwin - uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + uses: cygwin/cygwin-install-action@f61179d72284ceddc397ed07ddb444d82bf9e559 # v5 with: packages: make install-dir: C:\cygwin64 From 53f82146df6a3728f464e04f1ea2714c47e24494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 28 Jan 2025 09:47:26 +0100 Subject: [PATCH 19/51] [CI] Use personal access token for backport workflow (#15378) The backport workflow by default doesn't have authorization to update workflow files in the repository (see https://github.com/korthout/backport-action/issues/368). But workflow file updates are common in backported PRs. As a workaround, we can use a personal access token with approriate permissions. --- .github/workflows/backport.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 97c2657fcdf5..577e4a554652 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -22,10 +22,12 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + token: ${{ secrets.BACKPORT_ACTION_GITHUB_PAT }} - name: Create backport PR uses: korthout/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 with: + github_token: ${{ secrets.BACKPORT_ACTION_GITHUB_PAT }} # Config README: https://github.com/korthout/backport-action#backport-action copy_labels_pattern: '^(breaking-change|security|topic:.*|kind:.*|platform:.*)$' copy_milestone: true From 80e051d5f3b6474696c488374b86a928263a4fb5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 28 Jan 2025 20:18:50 +0100 Subject: [PATCH 20/51] Add `Crystal.once_init` replacing `__crystal_once_init` (#15371) Adds `Crystal.once_init` which is explictly called by `Crystal.init_runtime` as a replacement for the `__crystal_once_init` fun injected by the compiler. `__crystal_once_init` is obsolete and only kept for the legacy implementation. This is one less compiler dependency on runtime. --- src/compiler/crystal/codegen/once.cr | 20 +++---- src/crystal/main.cr | 3 +- src/crystal/once.cr | 88 ++++++++++++++-------------- 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/src/compiler/crystal/codegen/once.cr b/src/compiler/crystal/codegen/once.cr index ba08f0e7958b..e84dd6b541e0 100644 --- a/src/compiler/crystal/codegen/once.cr +++ b/src/compiler/crystal/codegen/once.cr @@ -5,20 +5,16 @@ class Crystal::CodeGenVisitor def once_init if once_init_fun = typed_fun?(@main_mod, ONCE_INIT) + # legacy (kept for backward compatibility): the compiler must save the + # state returned by __crystal_once_init once_init_fun = check_main_fun ONCE_INIT, once_init_fun - if once_init_fun.type.return_type.void? - call once_init_fun - else - # legacy (kept for backward compatibility): the compiler must save the - # state returned by __crystal_once_init - once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) - once_state_global.linkage = LLVM::Linkage::Internal if @single_module - once_state_global.initializer = once_init_fun.type.return_type.null - - state = call once_init_fun - store state, once_state_global - end + once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) + once_state_global.linkage = LLVM::Linkage::Internal if @single_module + once_state_global.initializer = once_init_fun.type.return_type.null + + state = call once_init_fun + store state, once_state_global end end diff --git a/src/crystal/main.cr b/src/crystal/main.cr index c26199fcdb4a..704153fe13f6 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -53,9 +53,10 @@ module Crystal # :nodoc: def self.init_runtime : Nil # `__crystal_once` directly or indirectly depends on `Fiber` and `Thread` - # so we explicitly initialize their class vars + # so we explicitly initialize their class vars, then init crystal/once Thread.init Fiber.init + Crystal.once_init end # :nodoc: diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 7b6c7e6ed0f2..87fa9147d56b 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -1,13 +1,13 @@ -# This file defines two functions expected by the compiler: +# This file defines the `__crystal_once` functions expected by the compiler. It +# is called each time a constant or class variable has to be initialized and is +# its responsibility to verify the initializer is executed only once and to fail +# on recursion. # -# - `__crystal_once_init`: executed only once at the beginning of the program -# and, for the legacy implementation, the result is passed on each call to -# `__crystal_once`. +# It also defines the `__crystal_once_init` function for backward compatibility +# with older compiler releases. It is executed only once at the beginning of the +# program and, for the legacy implementation, the result is passed on each call +# to `__crystal_once`. # -# - `__crystal_once`: called each time a constant or class variable has to be -# initialized and is its responsibility to verify the initializer is executed -# only once and to fail on recursion. - # In multithread mode a mutex is used to avoid race conditions between threads. # # On Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new @@ -28,12 +28,15 @@ {% if flag?(:preview_mt) || flag?(:win32) %} @@once_mutex = uninitialized Mutex - - # :nodoc: - def self.once_mutex=(@@once_mutex : Mutex) - end {% end %} + # :nodoc: + def self.once_init : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex = Mutex.new(:reentrant) + {% end %} + end + # :nodoc: # Using @[NoInline] so LLVM optimizes for the hot path (var already # initialized). @@ -67,13 +70,6 @@ end end - # :nodoc: - fun __crystal_once_init : Nil - {% if flag?(:preview_mt) || flag?(:win32) %} - Crystal.once_mutex = Mutex.new(:reentrant) - {% end %} - end - # :nodoc: # # Using `@[AlwaysInline]` allows LLVM to optimize const accesses. Since this @@ -94,37 +90,43 @@ # This implementation uses a global array to store the initialization flag # pointers for each value to find infinite loops and raise an error. - # :nodoc: - class Crystal::OnceState - @rec = [] of Bool* - - @[NoInline] - def once(flag : Bool*, initializer : Void*) - unless flag.value - if @rec.includes?(flag) - raise "Recursion while initializing class variables and/or constants" - end - @rec << flag - - Proc(Nil).new(initializer, Pointer(Void).null).call - flag.value = true - - @rec.pop - end - end - - {% if flag?(:preview_mt) || flag?(:win32) %} - @mutex = Mutex.new(:reentrant) + module Crystal + # :nodoc: + class OnceState + @rec = [] of Bool* @[NoInline] def once(flag : Bool*, initializer : Void*) unless flag.value - @mutex.synchronize do - previous_def + if @rec.includes?(flag) + raise "Recursion while initializing class variables and/or constants" end + @rec << flag + + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = true + + @rec.pop end end - {% end %} + + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + + @[NoInline] + def once(flag : Bool*, initializer : Void*) + unless flag.value + @mutex.synchronize do + previous_def + end + end + end + {% end %} + end + + # :nodoc: + def self.once_init : Nil + end end # :nodoc: From ea98639e02c00cef1304c84c2e0fcb1ca22a315d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:20:34 +0100 Subject: [PATCH 21/51] Fix: abstract EventLoop::Polling#system_add invalid signature (#15380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The abstract method refers to the non existing `Index` type. Weirdly the compiler won't complain until someone defines an `Index` type. Resolves #15357 (cherry picked from commit 5a245d905717ff9ea8a62512d8a5e0fefa721959) Co-authored-by: Johannes Müller --- src/crystal/event_loop/polling.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 2fe86ad5b649..3eb17c0e313e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -515,7 +515,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop private abstract def system_run(blocking : Bool, & : Fiber ->) : Nil # Add *fd* to the polling system, setting *index* as user data. - protected abstract def system_add(fd : Int32, index : Index) : Nil + protected abstract def system_add(fd : Int32, index : Arena::Index) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. # From 0ba7a4d6e951b35060ab1a4a3c3325d60352f16b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:21:01 +0100 Subject: [PATCH 22/51] Fix code example in `Process::Status#exit_code` docs (#15381) (cherry picked from commit 2b544f19eb687faa66e210127bb7c8fb75e65167) Co-authored-by: Billy.Zheng --- src/process/status.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/process/status.cr b/src/process/status.cr index 268d2f9e52d6..78cff49f0dc9 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -245,7 +245,7 @@ class Process::Status # Raises `RuntimeError` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code # => 1 + # Process.run("true").exit_code # => 0 # Process.run("exit 123", shell: true).exit_code # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code # ``` @@ -258,7 +258,7 @@ class Process::Status # Returns `nil` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code? # => 1 + # Process.run("true").exit_code? # => 0 # Process.run("exit 123", shell: true).exit_code? # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil # ``` From 6b1f07cd9f8d7be231da4bd17a50169507648c26 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:21:39 +0100 Subject: [PATCH 23/51] Fix GC `sig_suspend`, `sig_resume` for `gc_none` (#15382) The gc_none interface doesn't define the `sig_suspend` nor `sig_resume` class methods. The program should still compile but commit 57017f6 improperly checks for the method existence, and the methods are always required and compilation fails. (cherry picked from commit 0fbdcc90f1e907995671a4387fcc22f957321d61) Co-authored-by: Julien Portalier --- src/crystal/system/unix/pthread.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 73aa2a652ca1..98629a70fbb6 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -269,16 +269,16 @@ module Crystal::System::Thread {% end %} def self.sig_suspend : ::Signal - if GC.responds_to?(:sig_suspend) - GC.sig_suspend + if (gc = GC).responds_to?(:sig_suspend) + gc.sig_suspend else ::Signal.new(SIG_SUSPEND) end end def self.sig_resume : ::Signal - if GC.responds_to?(:sig_resume) - GC.sig_resume + if (gc = GC).responds_to?(:sig_resume) + gc.sig_resume else ::Signal.new(SIG_RESUME) end From a9e4821b9427469d27b0f4b78b5a4a4b6947508b Mon Sep 17 00:00:00 2001 From: Crys <159441349+crysbot@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:22:38 +0100 Subject: [PATCH 24/51] Update distribution-scripts (#15368) (#15385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 659cf2570d0c1a79486c8f1c54e41656eaca4e12) Co-authored-by: Johannes Müller --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 232ecb3d83b8..233c13068988 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "2f56eecf233f52229b93eab4acd2bca51f0f8edf" + default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 2978cd157a7a24a68abcc6d41a12dd7f04afc1f5 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Wed, 29 Jan 2025 07:31:53 -0500 Subject: [PATCH 25/51] Add docs to enum member helper methods (#15379) --- spec/compiler/semantic/enum_spec.cr | 24 +++++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/spec/compiler/semantic/enum_spec.cr b/spec/compiler/semantic/enum_spec.cr index 876694b99821..cf844b9711bd 100644 --- a/spec/compiler/semantic/enum_spec.cr +++ b/spec/compiler/semantic/enum_spec.cr @@ -613,4 +613,28 @@ describe "Semantic: enum" do a_def = result.program.types["Foo"].lookup_defs("foo").first a_def.previous.should be_nil end + + it "adds docs to helper methods" do + result = top_level_semantic <<-CRYSTAL, wants_doc: true + enum Foo + # These are the docs for `Bar` + Bar = 1 + end + CRYSTAL + + a_defs = result.program.types["Foo"].lookup_defs("bar?") + a_defs.first.doc.should eq("Returns `true` if this enum value equals `Bar`") + end + + it "marks helper methods with `:nodoc:` if the member is `:nodoc:`" do + result = top_level_semantic <<-CRYSTAL, wants_doc: true + enum Foo + # :nodoc: + Bar = 1 + end + CRYSTAL + + a_defs = result.program.types["Foo"].lookup_defs("bar?") + a_defs.first.doc.should eq(":nodoc:") + end end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 3654e24ff7a5..cfc8dddc81f1 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -824,6 +824,13 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor method_name = is_flags ? "includes?" : "==" body = Call.new(Var.new("self").at(member), method_name, Path.new(member.name).at(member)).at(member) a_def = Def.new("#{member.name.underscore}?", body: body).at(member) + + a_def.doc = if member.doc.try &.starts_with?(":nodoc:") + ":nodoc:" + else + "Returns `true` if this enum value #{is_flags ? "contains" : "equals"} `#{member.name}`" + end + enum_type.add_def a_def end From bd2f6b381a0911617def0eae195f8d21b5d1ce63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 30 Jan 2025 10:16:14 +0100 Subject: [PATCH 26/51] Update shards 0.19.1 (#15384) (cherry picked from commit 2697a0f) --- .github/workflows/win_build_portable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 585b9e67bd6a..bab121cc17f0 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 with: repository: crystal-lang/shards - ref: v0.19.0 + ref: v0.19.1 path: shards - name: Build shards release From 204b6a9f9ee7269a7bafe2f41c8d5a625b4acc19 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 1 Feb 2025 11:43:56 +0100 Subject: [PATCH 27/51] libcrypto: Correct EVP_CIPHER_get_flags argument type (#15392) Per OpenSSL documentation [^1] and the usage of `LibCrypto#evp_cipher_flags`, `EVP_CIPHER_get_flags` should accept `EVP_CIPHER` rather than `EVP_CIPHER_CTX`. An example of this usage can be seen in `OpenSSL::Cipher#authenticated?`. This change does not introduce any functional impact but improves clarity by correctly reflecting the expected argument, reducing the need for developers to cross-check other sources. [^1]: https://docs.openssl.org/1.1.1/man3/EVP_EncryptInit/#synopsis --- src/openssl/lib_crypto.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index 8e24bbcbc78e..b75474951764 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -246,14 +246,14 @@ lib LibCrypto fun evp_cipher_block_size = EVP_CIPHER_get_block_size(cipher : EVP_CIPHER) : Int32 fun evp_cipher_key_length = EVP_CIPHER_get_key_length(cipher : EVP_CIPHER) : Int32 fun evp_cipher_iv_length = EVP_CIPHER_get_iv_length(cipher : EVP_CIPHER) : Int32 - fun evp_cipher_flags = EVP_CIPHER_get_flags(ctx : EVP_CIPHER_CTX) : CipherFlags + fun evp_cipher_flags = EVP_CIPHER_get_flags(cipher : EVP_CIPHER) : CipherFlags {% else %} fun evp_cipher_name = EVP_CIPHER_name(cipher : EVP_CIPHER) : UInt8* fun evp_cipher_nid = EVP_CIPHER_nid(cipher : EVP_CIPHER) : Int32 fun evp_cipher_block_size = EVP_CIPHER_block_size(cipher : EVP_CIPHER) : Int32 fun evp_cipher_key_length = EVP_CIPHER_key_length(cipher : EVP_CIPHER) : Int32 fun evp_cipher_iv_length = EVP_CIPHER_iv_length(cipher : EVP_CIPHER) : Int32 - fun evp_cipher_flags = EVP_CIPHER_flags(ctx : EVP_CIPHER_CTX) : CipherFlags + fun evp_cipher_flags = EVP_CIPHER_flags(cipher : EVP_CIPHER) : CipherFlags {% end %} fun evp_cipher_ctx_new = EVP_CIPHER_CTX_new : EVP_CIPHER_CTX From 31e1b084432ed360544c429d8b5983cc14f831fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:08:40 +0100 Subject: [PATCH 28/51] Update distribution-scripts (#15388) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/1ee31a42f0b06776a42fa4635b54dc9ec567e68a This includes the following changes: * crystal-lang/distribution-scripts#350 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 233c13068988..9a0c64ce2400 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" + default: "1ee31a42f0b06776a42fa4635b54dc9ec567e68a" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 0c3c0fbd66d1cbb6ffb33b3f1695049ffa8b7d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:08:57 +0100 Subject: [PATCH 29/51] [CI] Add check for shards binary in `test_dist_linux_on_docker` (#15394) This is to detect issues like https://github.com/crystal-lang/distribution-scripts/issues/330 directly in the build process. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a0c64ce2400..49f5aa10df81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -477,6 +477,7 @@ jobs: - run: bin/ci prepare_system - run: echo 'export CURRENT_TAG="$CIRCLE_TAG"' >> $BASH_ENV - run: bin/ci prepare_build + - run: bin/ci with_build_env 'shards --version' - run: command: bin/ci build no_output_timeout: 30m From 3cc12a83c303848a11265f51b77bad3154fcfbaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:11:08 +0100 Subject: [PATCH 30/51] Disable directory path redirect when `directory_listing=false` (#15393) With `directory_listing: false` the handler returns 404 Not Found when requesting a directory with the trailing slash (`/foo/`). But, if the trailing slash is missing (`/foo`), it redirects to `/foo/` with a 302 Found response. This leaks information about the existence of directories and can lead to redirect loops if a reverse proxy tries to remove trailing slashes. Co-authored-by: Julien Portalier --- .../http/server/handlers/static_file_handler_spec.cr | 11 +++++++++++ src/http/server/handlers/static_file_handler.cr | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spec/std/http/server/handlers/static_file_handler_spec.cr b/spec/std/http/server/handlers/static_file_handler_spec.cr index 036e53eef2cc..dc87febc9e43 100644 --- a/spec/std/http/server/handlers/static_file_handler_spec.cr +++ b/spec/std/http/server/handlers/static_file_handler_spec.cr @@ -425,6 +425,17 @@ describe HTTP::StaticFileHandler do response.status_code.should eq(404) end + it "does not redirect directory when directory_listing=false" do + response = handle HTTP::Request.new("GET", "/foo"), directory_listing: false + response.status_code.should eq(404) + end + + it "redirect directory when directory_listing=true" do + response = handle HTTP::Request.new("GET", "/foo"), directory_listing: true + response.status_code.should eq(302) + response.headers["Location"].should eq "/foo/" + end + it "does not serve a not found file" do response = handle HTTP::Request.new("GET", "/not_found_file.txt") response.status_code.should eq(404) diff --git a/src/http/server/handlers/static_file_handler.cr b/src/http/server/handlers/static_file_handler.cr index cba0ff993ad2..665f466c7e46 100644 --- a/src/http/server/handlers/static_file_handler.cr +++ b/src/http/server/handlers/static_file_handler.cr @@ -68,7 +68,7 @@ class HTTP::StaticFileHandler file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native)) file_info = File.info? file_path - is_dir = file_info && file_info.directory? + is_dir = @directory_listing && file_info && file_info.directory? is_file = file_info && file_info.file? if request_path != expanded_path || is_dir && !is_dir_path @@ -85,7 +85,7 @@ class HTTP::StaticFileHandler context.response.headers["Accept-Ranges"] = "bytes" - if @directory_listing && is_dir + if is_dir context.response.content_type = "text/html; charset=utf-8" directory_listing(context.response, request_path, file_path) elsif is_file From a1d9b2210cada6bdfdf74617b4272db4be8c2c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:19:28 +0100 Subject: [PATCH 31/51] Add backports to changelog generator (#15402) Adds support for the automated backport workflow added in #15372. --- scripts/github-changelog.cr | 48 +++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index cc2f24a1f365..4d48e580a2c8 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,6 +271,18 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end + + def clean_title + title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") + end + + def backported? + labels.any?(&.starts_with?("backport")) + end + + def backport? + title.includes?("[Backport ") + end end def query_milestone(api_token, repository, number) @@ -312,8 +324,9 @@ end milestone = query_milestone(api_token, repository, milestone) -struct ChangelogEntry +class ChangelogEntry getter pull_requests : Array(PullRequest) + property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -342,13 +355,18 @@ struct ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") + io << pr.clean_title io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end + if backported_from = self.backported_from + io << ", backported from " + backported_from.link_ref(io) + end + authors = collect_authors if authors.present? io << ", thanks " @@ -361,15 +379,26 @@ struct ChangelogEntry def collect_authors authors = [] of String - pull_requests.each do |pr| + + if backported_from = self.backported_from + if author = backported_from.author + authors << author + end + end + + pull_requests.each_with_index do |pr, i| + next if backported_from && i.zero? + author = pr.author || next authors << author unless authors.includes?(author) end + authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } + backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -380,7 +409,7 @@ struct ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? + ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? end milestone.pull_requests.each do |pr| @@ -394,6 +423,17 @@ milestone.pull_requests.each do |pr| end end +milestone.pull_requests.each do |pr| + next unless pr.backported? + + backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } + if backport + backport.backported_from = pr + else + STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" + end +end + sections = entries.group_by(&.pr.section) SECTION_TITLES = { From 15cade627940285d61a8407c42905bcc9a8c1030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:20:33 +0100 Subject: [PATCH 32/51] Changelog for 1.15.1 (#15401) --- CHANGELOG.md | 46 +++++++++++++++++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 + src/VERSION | 2 +- 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d0c60fce07..f057833b46d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,51 @@ # Changelog +## [1.15.1] (2025-02-04) + +[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 + +### Bugfixes + +#### stdlib + +- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) +- *(runtime)* **[regression]** Fix: abstract `EventLoop::Polling#system_add` invalid signature ([#15380], thanks @github-actions) +- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) + +[#15393]: https://github.com/crystal-lang/crystal/pull/15393 +[#15380]: https://github.com/crystal-lang/crystal/pull/15380 +[#15382]: https://github.com/crystal-lang/crystal/pull/15382 +[#15349]: https://github.com/crystal-lang/crystal/pull/15349 + +### Documentation + +#### stdlib + +- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) + +[#15381]: https://github.com/crystal-lang/crystal/pull/15381 +[#15351]: https://github.com/crystal-lang/crystal/pull/15351 + +### Infrastructure + +- Changelog for 1.15.1 ([#15401], thanks @straight-shoota) +- Add backports to changelog generator ([#15402], thanks @straight-shoota) +- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) +- Update distribution-scripts ([#15388], thanks @straight-shoota) +- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) +- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) +- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) + +[#15401]: https://github.com/crystal-lang/crystal/pull/15401 +[#15402]: https://github.com/crystal-lang/crystal/pull/15402 +[#15385]: https://github.com/crystal-lang/crystal/pull/15385 +[#15368]: https://github.com/crystal-lang/crystal/pull/15368 +[#15388]: https://github.com/crystal-lang/crystal/pull/15388 +[#15344]: https://github.com/crystal-lang/crystal/pull/15344 +[#15384]: https://github.com/crystal-lang/crystal/pull/15384 +[#15366]: https://github.com/crystal-lang/crystal/pull/15366 +[#15394]: https://github.com/crystal-lang/crystal/pull/15394 + ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/shard.yml b/shard.yml index e979cf04bbec..5104666e2ff0 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.16.0-dev +version: 1.15.1 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH new file mode 100644 index 000000000000..e200f06cd97f --- /dev/null +++ b/src/SOURCE_DATE_EPOCH @@ -0,0 +1 @@ +1738627200 diff --git a/src/VERSION b/src/VERSION index 1f0d2f335194..ace44233b4aa 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.16.0-dev +1.15.1 From 87faf5a813a6811a414a402dd8d3cdd75133d792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:41:57 +0100 Subject: [PATCH 33/51] Revert "Changelog for 1.15.1 (#15401)" (#15407) This reverts commit 15cade627940285d61a8407c42905bcc9a8c1030. --- CHANGELOG.md | 46 ------------------------------------------- shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 4 files changed, 2 insertions(+), 49 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index f057833b46d7..a3d0c60fce07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,51 +1,5 @@ # Changelog -## [1.15.1] (2025-02-04) - -[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 - -### Bugfixes - -#### stdlib - -- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) -- *(runtime)* **[regression]** Fix: abstract `EventLoop::Polling#system_add` invalid signature ([#15380], thanks @github-actions) -- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) - -[#15393]: https://github.com/crystal-lang/crystal/pull/15393 -[#15380]: https://github.com/crystal-lang/crystal/pull/15380 -[#15382]: https://github.com/crystal-lang/crystal/pull/15382 -[#15349]: https://github.com/crystal-lang/crystal/pull/15349 - -### Documentation - -#### stdlib - -- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) - -[#15381]: https://github.com/crystal-lang/crystal/pull/15381 -[#15351]: https://github.com/crystal-lang/crystal/pull/15351 - -### Infrastructure - -- Changelog for 1.15.1 ([#15401], thanks @straight-shoota) -- Add backports to changelog generator ([#15402], thanks @straight-shoota) -- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) -- Update distribution-scripts ([#15388], thanks @straight-shoota) -- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) -- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) -- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) - -[#15401]: https://github.com/crystal-lang/crystal/pull/15401 -[#15402]: https://github.com/crystal-lang/crystal/pull/15402 -[#15385]: https://github.com/crystal-lang/crystal/pull/15385 -[#15368]: https://github.com/crystal-lang/crystal/pull/15368 -[#15388]: https://github.com/crystal-lang/crystal/pull/15388 -[#15344]: https://github.com/crystal-lang/crystal/pull/15344 -[#15384]: https://github.com/crystal-lang/crystal/pull/15384 -[#15366]: https://github.com/crystal-lang/crystal/pull/15366 -[#15394]: https://github.com/crystal-lang/crystal/pull/15394 - ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/shard.yml b/shard.yml index 5104666e2ff0..e979cf04bbec 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.1 +version: 1.16.0-dev authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index e200f06cd97f..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1738627200 diff --git a/src/VERSION b/src/VERSION index ace44233b4aa..1f0d2f335194 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.1 +1.16.0-dev From 89944bf1774f5a27c7518ccb77ff21b05918177b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 18:37:18 +0100 Subject: [PATCH 34/51] Changelog for 1.15.1 (#15406) --- CHANGELOG.md | 47 ++++++++++++++++++++++++++++++++++++ scripts/github-changelog.cr | 48 ++++--------------------------------- shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 5 files changed, 54 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d0c60fce07..904693af5f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## [1.15.1] (2025-02-04) + +[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 + +### Bugfixes + +#### stdlib + +- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) +- *(runtime)* **[regression]** abstract `EventLoop::Polling#system_add` invalid signature ([#15380], backported from [#15358], thanks @straight-shoota) +- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) + +[#15393]: https://github.com/crystal-lang/crystal/pull/15393 +[#15380]: https://github.com/crystal-lang/crystal/pull/15380 +[#15358]: https://github.com/crystal-lang/crystal/pull/15358 +[#15382]: https://github.com/crystal-lang/crystal/pull/15382 +[#15349]: https://github.com/crystal-lang/crystal/pull/15349 + +### Documentation + +#### stdlib + +- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) + +[#15381]: https://github.com/crystal-lang/crystal/pull/15381 +[#15351]: https://github.com/crystal-lang/crystal/pull/15351 + +### Infrastructure + +- Changelog for 1.15.1 ([#15406], thanks @straight-shoota) +- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) +- Update distribution-scripts ([#15388], thanks @straight-shoota) +- Add backports to changelog generator ([#15402], thanks @straight-shoota) +- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) +- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) +- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) + +[#15406]: https://github.com/crystal-lang/crystal/pull/15406 +[#15385]: https://github.com/crystal-lang/crystal/pull/15385 +[#15368]: https://github.com/crystal-lang/crystal/pull/15368 +[#15388]: https://github.com/crystal-lang/crystal/pull/15388 +[#15402]: https://github.com/crystal-lang/crystal/pull/15402 +[#15344]: https://github.com/crystal-lang/crystal/pull/15344 +[#15384]: https://github.com/crystal-lang/crystal/pull/15384 +[#15366]: https://github.com/crystal-lang/crystal/pull/15366 +[#15394]: https://github.com/crystal-lang/crystal/pull/15394 + ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index 4d48e580a2c8..cc2f24a1f365 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,18 +271,6 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end - - def clean_title - title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") - end - - def backported? - labels.any?(&.starts_with?("backport")) - end - - def backport? - title.includes?("[Backport ") - end end def query_milestone(api_token, repository, number) @@ -324,9 +312,8 @@ end milestone = query_milestone(api_token, repository, milestone) -class ChangelogEntry +struct ChangelogEntry getter pull_requests : Array(PullRequest) - property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -355,18 +342,13 @@ class ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.clean_title + io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end - if backported_from = self.backported_from - io << ", backported from " - backported_from.link_ref(io) - end - authors = collect_authors if authors.present? io << ", thanks " @@ -379,26 +361,15 @@ class ChangelogEntry def collect_authors authors = [] of String - - if backported_from = self.backported_from - if author = backported_from.author - authors << author - end - end - - pull_requests.each_with_index do |pr, i| - next if backported_from && i.zero? - + pull_requests.each do |pr| author = pr.author || next authors << author unless authors.includes?(author) end - authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } - backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -409,7 +380,7 @@ class ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? + ChangelogEntry.new(pr) unless pr.fixup? end milestone.pull_requests.each do |pr| @@ -423,17 +394,6 @@ milestone.pull_requests.each do |pr| end end -milestone.pull_requests.each do |pr| - next unless pr.backported? - - backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } - if backport - backport.backported_from = pr - else - STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" - end -end - sections = entries.group_by(&.pr.section) SECTION_TITLES = { diff --git a/shard.yml b/shard.yml index f0aef072ed4d..34953645d65e 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.0 +version: 1.15.1 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index 6fcaded9e558..e200f06cd97f 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1736380800 +1738627200 diff --git a/src/VERSION b/src/VERSION index 141f2e805beb..ace44233b4aa 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.0 +1.15.1 From 401443e2089a8a81945652f45d07161663918021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 5 Feb 2025 10:12:23 +0100 Subject: [PATCH 35/51] Fix `Union.from_yaml` to prioritize `String` for quoted scalar (#15405) --- spec/std/yaml/serialization_spec.cr | 12 ++++++++++++ src/yaml/enums.cr | 4 ++++ src/yaml/from_yaml.cr | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/spec/std/yaml/serialization_spec.cr b/spec/std/yaml/serialization_spec.cr index 414d44541fab..de0a111b7a86 100644 --- a/spec/std/yaml/serialization_spec.cr +++ b/spec/std/yaml/serialization_spec.cr @@ -390,6 +390,18 @@ describe "YAML serialization" do Bytes.from_yaml("!!binary aGVsbG8=").should eq("hello".to_slice) end + describe "Union.from_yaml" do + it "String priorization" do + (Int32 | String).from_yaml(%(42)).should eq 42 + (Int32 | String).from_yaml(%("42")).should eq "42" + + (String | UInt32).from_yaml(%(42)).should eq 42 + (String | UInt32).from_yaml(%("42")).should eq "42" + + (Int32 | UInt32).from_yaml(%("42")).should eq 42 + end + end + describe "parse exceptions" do it "has correct location when raises in Nil#from_yaml" do ex = expect_raises(YAML::ParseException) do diff --git a/src/yaml/enums.cr b/src/yaml/enums.cr index 2ab6789e0a4c..bf1d44fa2043 100644 --- a/src/yaml/enums.cr +++ b/src/yaml/enums.cr @@ -20,6 +20,10 @@ module YAML DOUBLE_QUOTED LITERAL FOLDED + + def quoted? + single_quoted? || double_quoted? + end end enum SequenceStyle diff --git a/src/yaml/from_yaml.cr b/src/yaml/from_yaml.cr index b9b6e7fae45c..227adb64c3c0 100644 --- a/src/yaml/from_yaml.cr +++ b/src/yaml/from_yaml.cr @@ -298,6 +298,13 @@ def Union.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # So, we give a chance first to types in the union to be parsed. {% string_type = T.find { |type| type == ::String } %} + {% if string_type %} + if node.as?(YAML::Nodes::Scalar).try(&.style.quoted?) + # do prefer String if it's a quoted scalar though + return String.new(ctx, node) + end + {% end %} + {% for type in T %} {% unless type == string_type %} begin From 1fafbb2bed6ab0b0d52ee2ad7706996cc4fad77b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 6 Feb 2025 03:15:12 +0800 Subject: [PATCH 36/51] Fix LLVM version detection for `-rc1` (#15410) The new [LLVM 20.1.0-rc1 release](https://github.com/llvm/llvm-project/releases/tag/llvmorg-20.1.0-rc1)'s `llvm-config` is a proper semantic version that has a hyphen before the `rc1` part, and needs to be stripped. --- src/llvm/lib_llvm.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 1349d5bf6a91..2ccaef0d2862 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -35,7 +35,7 @@ @[Link(ldflags: {{ llvm_ldflags }})] lib LibLLVM - VERSION = {{ llvm_version.strip.gsub(/git/, "").gsub(/rc.*/, "") }} + VERSION = {{ llvm_version.strip.gsub(/git/, "").gsub(/-?rc.*/, "") }} BUILT_TARGETS = {{ llvm_targets.strip.downcase.split(' ').map(&.id.symbolize) }} end {% end %} From 1c03dfa3df430942d8eac8273ed397c9a094b816 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 6 Feb 2025 03:15:27 +0800 Subject: [PATCH 37/51] Support LLVM 20 (#15412) Tested using version `20.1.0~++20250202013302+1eb7f4e6b461-1~exp1~20250202013426.11` from the official Apt repository, a few days beyond the RC1 release. Since Crystal has never used the MMX register type during LLVM IR generation, no major changes are needed. --- src/llvm/enums.cr | 7 ++++++- src/llvm/ext/llvm-versions.txt | 2 +- src/llvm/function_collection.cr | 8 +++++++- src/llvm/global_collection.cr | 8 +++++++- src/llvm/lib_llvm.cr | 1 + src/llvm/lib_llvm/core.cr | 12 ++++++++++-- 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/llvm/enums.cr b/src/llvm/enums.cr index ac23fa711560..7fb7c59b60df 100644 --- a/src/llvm/enums.cr +++ b/src/llvm/enums.cr @@ -249,7 +249,12 @@ module LLVM Pointer Vector Metadata - X86_MMX + X86_MMX # deleted in LLVM 20 + Token + ScalableVector + BFloat + X86_AMX + TargetExt end end diff --git a/src/llvm/ext/llvm-versions.txt b/src/llvm/ext/llvm-versions.txt index 6f4d3d4816d0..a5d4cfac2515 100644 --- a/src/llvm/ext/llvm-versions.txt +++ b/src/llvm/ext/llvm-versions.txt @@ -1 +1 @@ -19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 +20.1 19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 diff --git a/src/llvm/function_collection.cr b/src/llvm/function_collection.cr index 62e2bd2e6fc2..d2fdb75a97a1 100644 --- a/src/llvm/function_collection.cr +++ b/src/llvm/function_collection.cr @@ -30,7 +30,13 @@ struct LLVM::FunctionCollection end def []?(name) - func = LibLLVM.get_named_function(@mod, name) + func = + {% if LibLLVM::IS_LT_200 %} + LibLLVM.get_named_function(@mod, name) + {% else %} + LibLLVM.get_named_function_with_length(@mod, name, name.bytesize) + {% end %} + func ? Function.new(func) : nil end diff --git a/src/llvm/global_collection.cr b/src/llvm/global_collection.cr index 06d27a98de5e..7b214aed34af 100644 --- a/src/llvm/global_collection.cr +++ b/src/llvm/global_collection.cr @@ -9,7 +9,13 @@ struct LLVM::GlobalCollection end def []?(name) - global = LibLLVM.get_named_global(@mod, name) + global = + {% if LibLLVM::IS_LT_200 %} + LibLLVM.get_named_global(@mod, name) + {% else %} + LibLLVM.get_named_global_with_length(@mod, name, name.bytesize) + {% end %} + global ? Value.new(global) : nil end diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 2ccaef0d2862..14dae405097b 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -71,6 +71,7 @@ IS_LT_170 = {{compare_versions(LibLLVM::VERSION, "17.0.0") < 0}} IS_LT_180 = {{compare_versions(LibLLVM::VERSION, "18.0.0") < 0}} IS_LT_190 = {{compare_versions(LibLLVM::VERSION, "19.0.0") < 0}} + IS_LT_200 = {{compare_versions(LibLLVM::VERSION, "20.0.0") < 0}} end {% end %} diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index ef7b8f10b567..1c5a580e6c5b 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -50,7 +50,11 @@ lib LibLLVM fun get_module_context = LLVMGetModuleContext(m : ModuleRef) : ContextRef fun add_function = LLVMAddFunction(m : ModuleRef, name : Char*, function_ty : TypeRef) : ValueRef - fun get_named_function = LLVMGetNamedFunction(m : ModuleRef, name : Char*) : ValueRef + {% if LibLLVM::IS_LT_200 %} + fun get_named_function = LLVMGetNamedFunction(m : ModuleRef, name : Char*) : ValueRef + {% else %} + fun get_named_function_with_length = LLVMGetNamedFunctionWithLength(m : ModuleRef, name : Char*, length : SizeT) : ValueRef + {% end %} fun get_first_function = LLVMGetFirstFunction(m : ModuleRef) : ValueRef fun get_next_function = LLVMGetNextFunction(fn : ValueRef) : ValueRef @@ -144,7 +148,11 @@ lib LibLLVM fun set_alignment = LLVMSetAlignment(v : ValueRef, bytes : UInt) fun add_global = LLVMAddGlobal(m : ModuleRef, ty : TypeRef, name : Char*) : ValueRef - fun get_named_global = LLVMGetNamedGlobal(m : ModuleRef, name : Char*) : ValueRef + {% if LibLLVM::IS_LT_200 %} + fun get_named_global = LLVMGetNamedGlobal(m : ModuleRef, name : Char*) : ValueRef + {% else %} + fun get_named_global_with_length = LLVMGetNamedGlobalWithLength(m : ModuleRef, name : Char*, length : SizeT) : ValueRef + {% end %} fun get_initializer = LLVMGetInitializer(global_var : ValueRef) : ValueRef fun set_initializer = LLVMSetInitializer(global_var : ValueRef, constant_val : ValueRef) fun is_thread_local = LLVMIsThreadLocal(global_var : ValueRef) : Bool From 0da17469fe1d3ea0078585707db8809722813aec Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Wed, 5 Feb 2025 14:21:20 -0500 Subject: [PATCH 38/51] Add `:showdoc:` directive for `private` and `protected` objects (RFC #0011) (#15337) --- .../crystal/tools/doc/directives_spec.cr | 109 ++++++++++++++++++ src/compiler/crystal/tools/doc/generator.cr | 15 ++- .../tools/doc/html/_method_detail.html | 2 +- src/compiler/crystal/tools/doc/html/type.html | 4 +- src/compiler/crystal/tools/doc/macro.cr | 4 + src/compiler/crystal/tools/doc/method.cr | 13 ++- src/compiler/crystal/tools/doc/type.cr | 16 ++- 7 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 spec/compiler/crystal/tools/doc/directives_spec.cr diff --git a/spec/compiler/crystal/tools/doc/directives_spec.cr b/spec/compiler/crystal/tools/doc/directives_spec.cr new file mode 100644 index 000000000000..2036ffbfb753 --- /dev/null +++ b/spec/compiler/crystal/tools/doc/directives_spec.cr @@ -0,0 +1,109 @@ +require "../../../spec_helper" + +describe Crystal::Doc::Generator do + context ":nodoc:" do + it "hides documentation from being generated for methods" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + # :nodoc: + # + # Some docs + def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + end + + it "hides documentation from being generated for classes" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + class Foo + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.must_include?(program.types["Foo"]).should be_false + end + end + + context ":showdoc:" do + it "shows documentation for private methods" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + # :showdoc: + # + # Some docs + private def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + a_def = generator.type(program.types["Foo"]).lookup_method("foo").not_nil! + a_def.doc.should eq("Some docs") + a_def.visibility.should eq("private") + end + + it "does not include documentation for methods within a :nodoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + class Foo + # :showdoc: + # + # Some docs + private def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + + # If namespace isn't included, don't need to check if the method is included + generator.must_include?(program.types["Foo"]).should be_false + end + + it "does not include documentation for private and protected methods and objects in a :showdoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :showdoc: + class Foo + # Some docs for `foo` + private def foo + end + + # Some docs for `bar` + protected def bar + end + + # Some docs for `Baz` + private class Baz + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + generator.type(program.types["Foo"]).lookup_method("bar").should be_nil + + generator.must_include?(generator.type(program.types["Foo"]).lookup_path("Baz")).should be_false + end + + it "doesn't show a method marked :nodoc: within a :showdoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :showdoc: + class Foo + # :nodoc: + # Some docs for `foo` + def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + end + end +end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 4c5988cccae5..e24d521e1cb4 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -134,7 +134,7 @@ class Crystal::Doc::Generator end def must_include?(type : Crystal::Type) - return false if type.private? + return false if type.private? && !showdoc?(type) return false if nodoc? type return true if crystal_builtin?(type) @@ -215,6 +215,15 @@ class Crystal::Doc::Generator nodoc? obj.doc.try &.strip end + def showdoc?(str : String?) : Bool + return false if !str || !@program.wants_doc? + str.starts_with?(":showdoc:") + end + + def showdoc?(obj : Crystal::Type) + showdoc?(obj.doc.try &.strip) + end + def crystal_builtin?(type) return false unless project_info.crystal_stdlib? # TODO: Enabling this allows links to `NoReturn` to work, but has two `NoReturn`s show up in the sidebar @@ -267,7 +276,7 @@ class Crystal::Doc::Generator types = [] of Constant parent.type.types?.try &.each_value do |type| - if type.is_a?(Const) && must_include?(type) && !type.private? + if type.is_a?(Const) && must_include?(type) && (!type.private? || showdoc?(type)) types << Constant.new(self, parent, type) end end @@ -296,7 +305,7 @@ class Crystal::Doc::Generator end def doc(obj : Type | Method | Macro | Constant) - doc = obj.doc + doc = obj.doc.try &.strip.lchop(":showdoc:").strip return if !doc && !has_doc_annotations?(obj) diff --git a/src/compiler/crystal/tools/doc/html/_method_detail.html b/src/compiler/crystal/tools/doc/html/_method_detail.html index 3fc3d5cd760b..1834796bc056 100644 --- a/src/compiler/crystal/tools/doc/html/_method_detail.html +++ b/src/compiler/crystal/tools/doc/html/_method_detail.html @@ -6,7 +6,7 @@

<% methods.each do |method| %>
- <%= method.abstract? ? "abstract " : "" %> + <%= method.abstract? ? "abstract " : "" %><%= method.visibility.try(&.+(" ")) %> <%= method.kind %><%= method.name %><%= method.args_to_html %> # diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index 10c7e51fedd3..4438ebb2b883 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -19,7 +19,9 @@

<% if type.program? %> <%= type.full_name.gsub("::", "::") %> <% else %> - <%= type.abstract? ? "abstract " : ""%><%= type.kind %> <%= type.full_name.gsub("::", "::") %> + + <%= type.abstract? ? "abstract " : ""%><%= type.visibility.try(&.+(" ")) %><%= type.kind %> + <%= type.full_name.gsub("::", "::") %> <% end %>

diff --git a/src/compiler/crystal/tools/doc/macro.cr b/src/compiler/crystal/tools/doc/macro.cr index 49b9c30795bc..629eccc2e225 100644 --- a/src/compiler/crystal/tools/doc/macro.cr +++ b/src/compiler/crystal/tools/doc/macro.cr @@ -54,6 +54,10 @@ class Crystal::Doc::Macro false end + def visibility + @type.visibility + end + def kind "macro " end diff --git a/src/compiler/crystal/tools/doc/method.cr b/src/compiler/crystal/tools/doc/method.cr index 069deb48ee61..c43309ddf9a0 100644 --- a/src/compiler/crystal/tools/doc/method.cr +++ b/src/compiler/crystal/tools/doc/method.cr @@ -43,7 +43,7 @@ class Crystal::Doc::Method # This docs not include the "Description copied from ..." banner # in case it's needed. def doc - doc_info.doc + doc_info.doc.try &.strip.lchop(":showdoc:").strip end # Returns the type this method's docs are copied from, but @@ -135,6 +135,16 @@ class Crystal::Doc::Method end end + def visibility + case @def.visibility + in .public? + in .protected? + "protected" + in .private? + "private" + end + end + def constructor? return false unless @class_method return true if name == "new" @@ -323,6 +333,7 @@ class Crystal::Doc::Method builder.field "doc", doc unless doc.nil? builder.field "summary", formatted_summary unless formatted_summary.nil? builder.field "abstract", abstract? + builder.field "visibility", visibility if visibility builder.field "args", args unless args.empty? builder.field "args_string", args_to_s unless args.empty? builder.field "args_html", args_to_html unless args.empty? diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 624c8f017fe7..15cd3d5f2172 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -81,6 +81,10 @@ class Crystal::Doc::Type @type.abstract? end + def visibility + @type.private? ? "private" : nil + end + def parents_of?(type) return false unless type @@ -181,7 +185,7 @@ class Crystal::Doc::Type defs = [] of Method @type.defs.try &.each do |def_name, defs_with_metadata| defs_with_metadata.each do |def_with_metadata| - next unless def_with_metadata.def.visibility.public? + next if !def_with_metadata.def.visibility.public? && !showdoc?(def_with_metadata.def) next unless @generator.must_include? def_with_metadata.def defs << method(def_with_metadata.def, false) @@ -192,6 +196,10 @@ class Crystal::Doc::Type end end + private def showdoc?(adef) + @generator.showdoc?(adef.doc.try &.strip) + end + private def sort_order(item) # Sort operators first, then alphanumeric (case-insensitive). {item.name[0].alphanumeric? ? 1 : 0, item.name.downcase} @@ -205,7 +213,7 @@ class Crystal::Doc::Type @type.metaclass.defs.try &.each_value do |defs_with_metadata| defs_with_metadata.each do |def_with_metadata| a_def = def_with_metadata.def - next unless a_def.visibility.public? + next if !def_with_metadata.def.visibility.public? && !showdoc?(def_with_metadata.def) body = a_def.body @@ -236,7 +244,9 @@ class Crystal::Doc::Type macros = [] of Macro @type.metaclass.macros.try &.each_value do |the_macros| the_macros.each do |a_macro| - if a_macro.visibility.public? && @generator.must_include? a_macro + next if !a_macro.visibility.public? && !showdoc?(a_macro) + + if @generator.must_include? a_macro macros << self.macro(a_macro) end end From 8823b75a4570101c94584f4f70c5feea48836db0 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 6 Feb 2025 12:12:17 +0100 Subject: [PATCH 39/51] Add `EventLoop#wait_readable`, `#wait_writable` methods (#15376) These methods in the `Crystal::EventLoop` interfaces wait on file descriptor or socket readiness without performing an actual read or write operation, which can be delegated to an external library. This provides a semi-public interface to work around #15374: ```crystal file_descriptor = IO::FileDescriptor.new(LibC.some_fd, blocking: false) event_loop = Crystal::EventLoop.current event_loop.wait_readable(file_descriptor) LibC.do_something(fd) ``` This is implemented for the polling event loops (epoll, kqueue) as well as libevent since these were straightforward. Windows is left unimplemented. It might be implemented with `WSAPoll` running in a thread or `ProcessSocketNotifications` to associate sockets to a completion port. See [Winsock socket state notifications](https://learn.microsoft.com/en-us/windows/win32/winsock/winsock-socket-state-notifications) for more details. Related to [RFC #0007](https://github.com/crystal-lang/rfcs/blob/main/text/0007-event_loop-refactor.md) and [RFC #0009](https://github.com/crystal-lang/rfcs/blob/main/text/0009-lifetime-event_loop.md). --- src/crystal/event_loop/file_descriptor.cr | 6 ++++ src/crystal/event_loop/iocp.cr | 22 ++++++++++++++ src/crystal/event_loop/libevent.cr | 36 ++++++++++++++++++++--- src/crystal/event_loop/polling.cr | 24 +++++++++++++++ src/crystal/event_loop/socket.cr | 6 ++++ src/crystal/event_loop/wasi.cr | 32 ++++++++++++++++++-- src/io/evented.cr | 14 ++------- 7 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/crystal/event_loop/file_descriptor.cr b/src/crystal/event_loop/file_descriptor.cr index 0304f7d9b969..633fa180db68 100644 --- a/src/crystal/event_loop/file_descriptor.cr +++ b/src/crystal/event_loop/file_descriptor.cr @@ -9,6 +9,9 @@ abstract class Crystal::EventLoop # Returns 0 when EOF is reached. abstract def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 + # Blocks the current fiber until the file descriptor is ready for read. + abstract def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + # Writes at least one byte from *slice* to the file descriptor. # # Blocks the current fiber if the file descriptor isn't ready for writing, @@ -17,6 +20,9 @@ abstract class Crystal::EventLoop # Returns the number of bytes written (up to `slice.size`). abstract def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 + # Blocks the current fiber until the file descriptor is ready for write. + abstract def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + # Closes the file descriptor resource. abstract def close(file_descriptor : Crystal::System::FileDescriptor) : Nil end diff --git a/src/crystal/event_loop/iocp.cr b/src/crystal/event_loop/iocp.cr index 6e4175e3daee..da827079312a 100644 --- a/src/crystal/event_loop/iocp.cr +++ b/src/crystal/event_loop/iocp.cr @@ -190,6 +190,10 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop end.to_i32 end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + raise NotImplementedError.new("Crystal::System::IOCP#wait_readable(FileDescriptor)") + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 System::IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| ret = LibC.WriteFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) @@ -197,6 +201,10 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop end.to_i32 end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + raise NotImplementedError.new("Crystal::System::IOCP#wait_writable(FileDescriptor)") + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil LibC.CancelIoEx(file_descriptor.windows_handle, nil) unless file_descriptor.system_blocking? end @@ -220,6 +228,13 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop bytes_read.to_i32 end + def wait_readable(socket : ::Socket) : Nil + # NOTE: Windows 10+ has `ProcessSocketNotifications` to associate sockets to + # a completion port and be notified of socket readiness. See + # + raise NotImplementedError.new("Crystal::System::IOCP#wait_readable(Socket)") + end + def write(socket : ::Socket, slice : Bytes) : Int32 wsabuf = wsa_buffer(slice) @@ -231,6 +246,13 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop bytes.to_i32 end + def wait_writable(socket : ::Socket) : Nil + # NOTE: Windows 10+ has `ProcessSocketNotifications` to associate sockets to + # a completion port and be notified of socket readiness. See + # + raise NotImplementedError.new("Crystal::System::IOCP#wait_writable(Socket)") + end + def send_to(socket : ::Socket, slice : Bytes, address : ::Socket::Address) : Int32 wsabuf = wsa_buffer(slice) bytes_written = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASendTo", socket.write_timeout) do |overlapped| diff --git a/src/crystal/event_loop/libevent.cr b/src/crystal/event_loop/libevent.cr index 9c0b3d33b15c..636d01331624 100644 --- a/src/crystal/event_loop/libevent.cr +++ b/src/crystal/event_loop/libevent.cr @@ -84,6 +84,12 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| @@ -94,6 +100,12 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil file_descriptor.evented_close end @@ -104,12 +116,24 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_readable(socket : ::Socket) : Nil + socket.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0).to_i32 end end + def wait_writable(socket : ::Socket) : Nil + socket.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def receive_from(socket : ::Socket, slice : Bytes) : Tuple(Int32, ::Socket::Address) sockaddr = Pointer(LibC::SockaddrStorage).malloc.as(LibC::Sockaddr*) # initialize sockaddr with the initialized family of the socket @@ -142,7 +166,7 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop when Errno::EISCONN return when Errno::EINPROGRESS, Errno::EALREADY - socket.wait_writable(timeout: timeout) do + socket.evented_wait_writable(timeout: timeout) do return IO::TimeoutError.new("connect timed out") end else @@ -174,7 +198,7 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop if socket.closed? return elsif Errno.value == Errno::EAGAIN - socket.wait_readable(raise_if_closed: false) do + socket.evented_wait_readable(raise_if_closed: false) do raise IO::TimeoutError.new("Accept timed out") end return if socket.closed? @@ -200,7 +224,9 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_readable + target.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end @@ -218,7 +244,9 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_writable + target.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 3eb17c0e313e..4df9eff7bc8e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -146,6 +146,12 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end end + def wait_readable(file_descriptor : System::FileDescriptor) : Nil + wait_readable(file_descriptor, file_descriptor.@read_timeout) do + raise IO::TimeoutError.new + end + end + def write(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 size = evented_write(file_descriptor, slice, file_descriptor.@write_timeout) @@ -160,6 +166,12 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end end + def wait_writable(file_descriptor : System::FileDescriptor) : Nil + wait_writable(file_descriptor, file_descriptor.@write_timeout) do + raise IO::TimeoutError.new + end + end + def close(file_descriptor : System::FileDescriptor) : Nil evented_close(file_descriptor) end @@ -176,12 +188,24 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop size end + def wait_readable(socket : ::Socket) : Nil + wait_readable(socket, socket.@read_timeout) do + raise IO::TimeoutError.new + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 size = evented_write(socket, slice, socket.@write_timeout) raise IO::Error.from_errno("write", target: socket) if size == -1 size end + def wait_writable(socket : ::Socket) : Nil + wait_writable(socket, socket.@write_timeout) do + raise IO::TimeoutError.new + end + end + def accept(socket : ::Socket) : ::Socket::Handle? loop do client_fd = diff --git a/src/crystal/event_loop/socket.cr b/src/crystal/event_loop/socket.cr index 1f4fc629d8ca..2e3679e615c5 100644 --- a/src/crystal/event_loop/socket.cr +++ b/src/crystal/event_loop/socket.cr @@ -15,6 +15,9 @@ abstract class Crystal::EventLoop # Use `#receive_from` for capturing the source address of a message. abstract def read(socket : ::Socket, slice : Bytes) : Int32 + # Blocks the current fiber until the socket is ready for read. + abstract def wait_readable(socket : ::Socket) : Nil + # Writes at least one byte from *slice* to the socket. # # Blocks the current fiber if the socket is not ready for writing, @@ -25,6 +28,9 @@ abstract class Crystal::EventLoop # Use `#send_to` for sending a message to a specific target address. abstract def write(socket : ::Socket, slice : Bytes) : Int32 + # Blocks the current fiber until the socket is ready for write. + abstract def wait_writable(socket : ::Socket) : Nil + # Accepts an incoming TCP connection on the socket. # # Blocks the current fiber if no connection is waiting, continuing when one diff --git a/src/crystal/event_loop/wasi.cr b/src/crystal/event_loop/wasi.cr index 08781b4fb950..028eb7e0e9a8 100644 --- a/src/crystal/event_loop/wasi.cr +++ b/src/crystal/event_loop/wasi.cr @@ -39,6 +39,12 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| @@ -49,6 +55,12 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_writable(raise_if_closed: false) do + raise IO::TimeoutError.new("Write timed out") + end + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil file_descriptor.evented_close end @@ -59,12 +71,24 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_readable(socket : ::Socket) : Nil + socket.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0) end end + def wait_writable(socket : ::Socket) : Nil + socket.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def receive_from(socket : ::Socket, slice : Bytes) : Tuple(Int32, ::Socket::Address) raise NotImplementedError.new "Crystal::Wasi::EventLoop#receive_from" end @@ -94,7 +118,9 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_readable + target.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end @@ -112,7 +138,9 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_writable + target.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end diff --git a/src/io/evented.cr b/src/io/evented.cr index 1f95d1870b0b..635c399d9239 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -34,12 +34,7 @@ module IO::Evented end # :nodoc: - def wait_readable(timeout = @read_timeout) : Nil - wait_readable(timeout: timeout) { raise TimeoutError.new("Read timed out") } - end - - # :nodoc: - def wait_readable(timeout = @read_timeout, *, raise_if_closed = true, &) : Nil + def evented_wait_readable(timeout = @read_timeout, *, raise_if_closed = true, &) : Nil readers = @readers.get { Deque(Fiber).new } readers << Fiber.current add_read_event(timeout) @@ -59,12 +54,7 @@ module IO::Evented end # :nodoc: - def wait_writable(timeout = @write_timeout) : Nil - wait_writable(timeout: timeout) { raise TimeoutError.new("Write timed out") } - end - - # :nodoc: - def wait_writable(timeout = @write_timeout, &) : Nil + def evented_wait_writable(timeout = @write_timeout, &) : Nil writers = @writers.get { Deque(Fiber).new } writers << Fiber.current add_write_event(timeout) From 2ee32df3cce83de3a6cd1626d71e3dff9315463b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 7 Feb 2025 14:19:37 +0100 Subject: [PATCH 40/51] [CI] Add test against LLVM 20 [fixup #15412] (#15418) --- .github/workflows/llvm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 35b44df4aae6..0d94ed82b418 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -25,6 +25,7 @@ jobs: - {llvm_version: 17, runs-on: ubuntu-24.04, codename: noble} - {llvm_version: 18, runs-on: ubuntu-24.04, codename: noble} - {llvm_version: 19, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 20, runs-on: ubuntu-24.04, codename: noble} name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source From 34a3f456749401c342c97e4766181bbd03f2ffcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 7 Feb 2025 14:23:20 +0100 Subject: [PATCH 41/51] Update previous release: Crystal 1.15.1 (#15417) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/mingw-w64.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3349a0b0a76d..70d16b6c280e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 59ea25fdce3c..9a6bc722cfa4 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -15,7 +15,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: build-interpreter: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build strategy: matrix: part: [0, 1, 2, 3] @@ -69,7 +69,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0b1c4f31d260..aff46c8d76be 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.1, 1.15.0] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.1, 1.15.1] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 0d94ed82b418..b71ef2bb7930 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -41,7 +41,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 632d151695df..a2a253247c6b 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -29,7 +29,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Cross-compile Crystal run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 990909299f80..efcf9ea2c150 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -12,7 +12,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 7819d9de6fd0..c87cb9058682 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -12,7 +12,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -27,7 +27,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index d60224fa1300..f5872035f780 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -14,7 +14,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-24.04 - container: crystallang/crystal:1.15.0-build + container: crystallang/crystal:1.15.1-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 601f1ad2eaa9..398705ed21b5 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -27,7 +27,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 25850d0bd6d0..c2ffba8f341d 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.15.0-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.15.1-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.15.0}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.15.1}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index 48139afeb6d3..2a9d3c7ed15a 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0lcx313gz11x2cjdzy9pbvs8z1ixf0vj9gbjqni10smxgziv4v8k"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0lcx313gz11x2cjdzy9pbvs8z1ixf0vj9gbjqni10smxgziv4v8k"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-linux-x86_64.tar.gz"; - sha256 = "sha256:14zxv6v19phb5ippn851g928w5sf9399ikilaxpiy3xjswsxwf07"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-linux-x86_64.tar.gz"; + sha256 = "sha256:1d0bl3sf3k7f4ns85vr7s7kb0hw2l333gpkvbzjw7ygb675016km"; }; }.${pkgs.stdenv.system}); From 5172d01233447585984c9ec77d614b1116fa5da2 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 7 Feb 2025 14:23:52 +0100 Subject: [PATCH 42/51] Fix: don't set external linkage when `@[NoInline]` is specified (#15424) Unlike `alwaysinline` that is treated by LLVM as a simple hint, the `noinline` function attribute is enough to tell LLVM to always generate the symbol and never inline the calls (even with aggressive optimizations). We don't need to set the linkage to external. --- src/compiler/crystal/codegen/fun.cr | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index c56bde6e5c2a..abe57df37aac 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -337,7 +337,7 @@ class Crystal::CodeGenVisitor end end - if @single_module && !target_def.no_inline? && !target_def.is_a?(External) + if @single_module && !target_def.is_a?(External) context.fun.linkage = LLVM::Linkage::Internal end @@ -448,11 +448,7 @@ class Crystal::CodeGenVisitor context.fun.add_attribute LLVM::Attribute::ReturnsTwice if target_def.returns_twice? context.fun.add_attribute LLVM::Attribute::Naked if target_def.naked? context.fun.add_attribute LLVM::Attribute::NoReturn if target_def.no_returns? - - if target_def.no_inline? - context.fun.add_attribute LLVM::Attribute::NoInline - context.fun.linkage = LLVM::Linkage::External - end + context.fun.add_attribute LLVM::Attribute::NoInline if target_def.no_inline? end def setup_closure_vars(def_vars, closure_vars, context, closure_type, closure_ptr) From 687dce2bb01d1f921f70f01a9fd8fae0aff07c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 8 Feb 2025 20:33:41 +0100 Subject: [PATCH 43/51] Fix `Tuple#to_a(&)` for arbitrary block output type (#15431) `Tuple#to_a(&)` is an optimized implementation of `Enumerable#to_a(&)` (overriding `Indexable#to_a(&)`). But the array is wrongly typed. It is impossible to map to a type that's not in the tuple: ```cr {1}.to_a(&.to_s) # Error: expected argument #2 to 'Pointer(Int32)#[]=' to be Int32, not String ``` Like the original implementations, `Tuple#to_a` must create an array that can hold the output type of the block because that's what it collects. It has nothing to do with the type of the tuple. --- spec/std/tuple_spec.cr | 28 ++++++++++++++++++++++------ src/tuple.cr | 10 +++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/spec/std/tuple_spec.cr b/spec/std/tuple_spec.cr index ec240234d8ed..31240a72fce1 100644 --- a/spec/std/tuple_spec.cr +++ b/spec/std/tuple_spec.cr @@ -333,13 +333,29 @@ describe "Tuple" do ({1, 2} === nil).should be_false end - it "does to_a" do - ary = {1, 'a', true}.to_a - ary.should eq([1, 'a', true]) - ary.size.should eq(3) + describe "#to_a" do + describe "without block" do + it "basic" do + ary = {1, 'a', true}.to_a + ary.should eq([1, 'a', true]) + ary.size.should eq(3) + end + + it "empty" do + ary = Tuple.new.to_a + ary.size.should eq(0) + end + end - ary = Tuple.new.to_a - ary.size.should eq(0) + describe "with block" do + it "basic" do + {-1, -2, -3}.to_a(&.abs).should eq [1, 2, 3] + end + + it "different type" do + {1, 2, true}.to_a(&.to_s).should eq ["1", "2", "true"] + end + end end # Tuple#to_static_array don't compile on aarch64-darwin and diff --git a/src/tuple.cr b/src/tuple.cr index 2f9cde352e4f..a658e36774c7 100644 --- a/src/tuple.cr +++ b/src/tuple.cr @@ -545,11 +545,7 @@ struct Tuple # {1, 2, 3, 4, 5}.to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(Union(*T)) - {% if compare_versions(Crystal::VERSION, "1.1.0") < 0 %} - to_a(&.itself.as(Union(*T))) - {% else %} - to_a(&.itself) - {% end %} + to_a(&.as(Union(*T))) end # Returns an `Array` with the results of running *block* against each element of the tuple. @@ -557,8 +553,8 @@ struct Tuple # ``` # {1, 2, 3, 4, 5}).to_a { |i| i * 2 } # => [2, 4, 6, 8, 10] # ``` - def to_a(& : Union(*T) -> _) - Array(Union(*T)).build(size) do |buffer| + def to_a(& : Union(*T) -> U) forall U + Array(U).build(size) do |buffer| {% for i in 0...T.size %} buffer[{{i}}] = yield self[{{i}}] {% end %} From 309029189254515de458e59222becccf6e37c7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 8 Feb 2025 20:34:40 +0100 Subject: [PATCH 44/51] Add backports to changelog generator (#15413) Adds support for the automated backport workflow added in #15372. --- scripts/github-changelog.cr | 48 +++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index cc2f24a1f365..4d48e580a2c8 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,6 +271,18 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end + + def clean_title + title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") + end + + def backported? + labels.any?(&.starts_with?("backport")) + end + + def backport? + title.includes?("[Backport ") + end end def query_milestone(api_token, repository, number) @@ -312,8 +324,9 @@ end milestone = query_milestone(api_token, repository, milestone) -struct ChangelogEntry +class ChangelogEntry getter pull_requests : Array(PullRequest) + property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -342,13 +355,18 @@ struct ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") + io << pr.clean_title io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end + if backported_from = self.backported_from + io << ", backported from " + backported_from.link_ref(io) + end + authors = collect_authors if authors.present? io << ", thanks " @@ -361,15 +379,26 @@ struct ChangelogEntry def collect_authors authors = [] of String - pull_requests.each do |pr| + + if backported_from = self.backported_from + if author = backported_from.author + authors << author + end + end + + pull_requests.each_with_index do |pr, i| + next if backported_from && i.zero? + author = pr.author || next authors << author unless authors.includes?(author) end + authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } + backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -380,7 +409,7 @@ struct ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? + ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? end milestone.pull_requests.each do |pr| @@ -394,6 +423,17 @@ milestone.pull_requests.each do |pr| end end +milestone.pull_requests.each do |pr| + next unless pr.backported? + + backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } + if backport + backport.backported_from = pr + else + STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" + end +end + sections = entries.group_by(&.pr.section) SECTION_TITLES = { From 0cc0264f423f136db5baa190118b722dade09681 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko <67241138+homonoidian@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:35:00 +0300 Subject: [PATCH 45/51] Implement `StringLiteral#scan` (#15398) --- spec/compiler/macro/macro_methods_spec.cr | 6 +++ src/compiler/crystal/macros.cr | 6 +++ src/compiler/crystal/macros/methods.cr | 54 +++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index a0884b8331e9..508ae594a7d2 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -571,6 +571,12 @@ module Crystal assert_macro %({{"hello".gsub(/e|o/, "a")}}), %("halla") end + it "executes scan" do + assert_macro %({{"Crystal".scan(/(Cr)(?y)(st)(?al)/)}}), %([{0 => "Crystal", 1 => "Cr", "name1" => "y", 3 => "st", "name2" => "al"} of ::Int32 | ::String => ::String | ::Nil] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + assert_macro %({{"Crystal".scan(/(Cr)?(stal)/)}}), %([{0 => "stal", 1 => nil, 2 => "stal"} of ::Int32 | ::String => ::String | ::Nil] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + assert_macro %({{"Ruby".scan(/Crystal/)}}), %([] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + end + it "executes camelcase" do assert_macro %({{"foo_bar".camelcase}}), %("FooBar") end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index ae6634e83a6f..0048bf635dcc 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -62,6 +62,12 @@ private macro def_string_methods(klass) def includes?(search : StringLiteral | CharLiteral) : BoolLiteral end + # Returns an array of capture hashes for each match of *regex* in this string. + # + # Capture hashes have the same form as `Regex::MatchData#to_h`. + def scan(regex : RegexLiteral) : ArrayLiteral(HashLiteral(NumberLiteral | StringLiteral), StringLiteral | NilLiteral) + end + # Similar to `String#size`. def size : NumberLiteral end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index ede6ebb28a65..24992816c8f6 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -758,6 +758,60 @@ module Crystal end BoolLiteral.new(@value.includes?(piece)) end + when "scan" + interpret_check_args do |arg| + unless arg.is_a?(RegexLiteral) + raise "StringLiteral#scan expects a regex, not #{arg.class_desc}" + end + + regex_value = arg.value + if regex_value.is_a?(StringLiteral) + regex = Regex.new(regex_value.value, arg.options) + else + raise "regex interpolations not yet allowed in macros" + end + + matches = ArrayLiteral.new( + of: Generic.new( + Path.global("Hash"), + [ + Union.new([Path.global("Int32"), Path.global("String")] of ASTNode), + Union.new([Path.global("String"), Path.global("Nil")] of ASTNode), + ] of ASTNode + ) + ) + + @value.scan(regex) do |match_data| + captures = HashLiteral.new( + of: HashLiteral::Entry.new( + Union.new([Path.global("Int32"), Path.global("String")] of ASTNode), + Union.new([Path.global("String"), Path.global("Nil")] of ASTNode), + ) + ) + + match_data.to_h.each do |capture, substr| + case capture + in Int32 + key = NumberLiteral.new(capture) + in String + key = StringLiteral.new(capture) + end + + case substr + in String + value = StringLiteral.new(substr) + in Nil + value = NilLiteral.new + end + + captures.entries << HashLiteral::Entry.new(key, value) + end + + matches.elements << captures + end + + matches + end when "size" interpret_check_args { NumberLiteral.new(@value.size) } when "lines" From a2573f9142bec8680bd81f08739a0cd665d4ad86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 9 Feb 2025 19:39:56 +0100 Subject: [PATCH 46/51] Add `Union.from_json_object_key?` (#15411) --- spec/std/json/serialization_spec.cr | 17 +++++++++++++++++ src/json/from_json.cr | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/spec/std/json/serialization_spec.cr b/spec/std/json/serialization_spec.cr index 80fc83e13b7e..a3dad00a7737 100644 --- a/spec/std/json/serialization_spec.cr +++ b/spec/std/json/serialization_spec.cr @@ -143,6 +143,23 @@ describe "JSON serialization" do Hash(BigDecimal, String).from_json(%({"1234567890.123456789": "x"})).should eq({"1234567890.123456789".to_big_d => "x"}) end + describe "Hash with union key (Union.from_json_object_key?)" do + it "string deprioritized" do + Hash(String | Int32, Nil).from_json(%({"1": null})).should eq({1 => nil}) + Hash(String | UInt32, Nil).from_json(%({"1": null})).should eq({1 => nil}) + end + + it "string without alternative" do + Hash(String | Int32, Nil).from_json(%({"foo": null})).should eq({"foo" => nil}) + end + + it "no match" do + expect_raises JSON::ParseException, %(Can't convert "foo" into (Float64 | Int32) at line 1, column 2) do + Hash(Float64 | Int32, Nil).from_json(%({"foo": null})) + end + end + end + it "raises an error Hash(String, Int32)#from_json with null value" do expect_raises(JSON::ParseException, "Expected Int but was Null") do Hash(String, Int32).from_json(%({"foo": 1, "bar": 2, "baz": null})) diff --git a/src/json/from_json.cr b/src/json/from_json.cr index 1c6a9e3c9c29..92edf0472c77 100644 --- a/src/json/from_json.cr +++ b/src/json/from_json.cr @@ -440,6 +440,28 @@ def Union.new(pull : JSON::PullParser) {% end %} end +def Union.from_json_object_key?(key : String) + {% begin %} + # String must come last because any key can be parsed into a String. + # So, we give a chance first to other types in the union to be parsed. + {% string_type = T.find { |type| type == ::String } %} + + {% for type in T %} + {% unless type == string_type %} + if result = {{ type }}.from_json_object_key?(key) + return result + end + {% end %} + {% end %} + + {% if string_type %} + if result = {{ string_type }}.from_json_object_key?(key) + return result + end + {% end %} + {% end %} +end + # Reads a string from JSON parser as a time formatted according to [RFC 3339](https://tools.ietf.org/html/rfc3339) # or other variations of [ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf). # From e4c904a92cd7e67b295e10c178bdbe6637679056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 12:00:25 +0100 Subject: [PATCH 47/51] Simplify `Enumerable#to_a` (#15432) The implementation of `#to_a` is identical to the yielding variant `#to_a(&)`. We can simply delegate to `#to_a(&)`. As a bonus, inheriting types which override `#to_a(&)` with an optimized implementation will implicitly use the same optimization for `#to_a` as well. This means we can drop `Indexable#to_a`. Inheriting types `Tuple` and `Hash` already explicitly delegate `#to_a` to their override implementations of `#to_a(&)`. We keep these overrides of `#to_a` because they augment the documentation. But we replace the method bodies with `super` to make it clear that the behaviour is inherited and the `def` only provides documentation. --- src/enumerable.cr | 4 +--- src/hash.cr | 2 +- src/indexable.cr | 11 ----------- src/tuple.cr | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/enumerable.cr b/src/enumerable.cr index 0993f38bbc4d..9fcba66ddf3a 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -2013,9 +2013,7 @@ module Enumerable(T) # (1..5).to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(T) - ary = [] of T - each { |e| ary << e } - ary + to_a(&.as(T)) end # Returns an `Array` with the results of running *block* against each element of the collection. diff --git a/src/hash.cr b/src/hash.cr index 1be6543d730c..c145bda36309 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2083,7 +2083,7 @@ class Hash(K, V) # # The order of the array follows the order the keys were inserted in the Hash. def to_a : Array({K, V}) - to_a(&.itself) + super end # Returns an `Array` with the results of running *block* against tuples with key and values diff --git a/src/indexable.cr b/src/indexable.cr index 4a3990e83870..3f6dca1762b1 100644 --- a/src/indexable.cr +++ b/src/indexable.cr @@ -693,17 +693,6 @@ module Indexable(T) end end - # Returns an `Array` with all the elements in the collection. - # - # ``` - # {1, 2, 3}.to_a # => [1, 2, 3] - # ``` - def to_a : Array(T) - ary = Array(T).new(size) - each { |e| ary << e } - ary - end - # Returns an `Array` with the results of running *block* against each element of the collection. # # ``` diff --git a/src/tuple.cr b/src/tuple.cr index a658e36774c7..a8dd3a040727 100644 --- a/src/tuple.cr +++ b/src/tuple.cr @@ -545,7 +545,7 @@ struct Tuple # {1, 2, 3, 4, 5}.to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(Union(*T)) - to_a(&.as(Union(*T))) + super end # Returns an `Array` with the results of running *block* against each element of the tuple. From 4f6b61ec81b6c2af8d6c844a1f9d1e1f1fcd0afa Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 10 Feb 2025 12:03:01 +0100 Subject: [PATCH 48/51] Move shadow space reservation to x86_64 `makecontext` (#15434) The shadow space (or home space) is a requirement of the x64 call convention: we must reserve 32 bytes before the return address on the stack. It only applies to x64, not to arm64 for example. We don't have to deal with this when creating the stack. This is a consideration for each individual `makecontext` to handle. ### Explanation The old version for win32: 1. aligned the ptr to 16 bytes; 2. decremented the ptr by 6 pointer-size (8 bytes): 1 to move it into the stack area (8 bytes), 4 for the shadow space (32 bytes), and 1 last to keep the 16-bytes alignment (another 8 bytes); 3. aligned the ptr to 16 bytes (not needed). The new version (actually the old non-win32 version): 1. decrements the ptr by 1 pointer-size to move it back _into_ the stack area; 2. aligns the ptr to 16 bytes. In both versions `stack_ptr` point to the ~~last~~ first (the stack grows down) addressable `Void*` in the stack and is aligned to 16 bytes. Then the new x86_64-microsoft variant of `makecontext` reserves 32 bytes of shadow space (still aligned to 16 bytes) and the rest of `makecontext` is still aligned to 16 bytes. ### References - [Microsoft: x64 call convention](https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170) - [Shadow Space and Stack Alignment (x64)](https://open-advanced-windows-exploitati.gitbook.io/open-advanced-windows-exploitation/custom-shellcode/64-bit-architecture/calling-conventions#shadow-space-and-stack-alignment) - [Microsoft: arm64 call convention](https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170) --- src/fiber.cr | 18 +++--------------- src/fiber/context/x86_64-microsoft.cr | 5 +++-- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/fiber.cr b/src/fiber.cr index 3d2fe9a89797..60162e5872a5 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -108,21 +108,9 @@ class Fiber fiber_main = ->(f : Fiber) { f.run } - # FIXME: This line shouldn't be necessary (#7975) - stack_ptr = nil - {% if flag?(:win32) %} - # align stack bottom to 16 bytes - @stack_bottom = Pointer(Void).new(@stack_bottom.address & ~0x0f_u64) - - # It's the caller's responsibility to allocate 32 bytes of "shadow space" on the stack right - # before calling the function (regardless of the actual number of parameters used) - - stack_ptr = @stack_bottom - sizeof(Void*) * 6 - {% else %} - # point to first addressable pointer on the stack (@stack_bottom points past - # the stack because the stack grows down): - stack_ptr = @stack_bottom - sizeof(Void*) - {% end %} + # point to first addressable pointer on the stack (@stack_bottom points past + # the stack because the stack grows down): + stack_ptr = @stack_bottom - sizeof(Void*) # align the stack pointer to 16 bytes: stack_ptr = Pointer(Void*).new(stack_ptr.address & ~0x0f_u64) diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 08576fc348aa..a1b9fa281074 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -6,14 +6,15 @@ class Fiber # A great explanation on stack contexts for win32: # https://web.archive.org/web/20220527113808/https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows - # 8 registers + 3 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers - @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) + # 4 shadow space + (8 registers + 3 qwords for NT_TIB + 1 parameter) + 10 128bit XMM registers + @context.stack_top = (stack_ptr - (4 + 12 + 10*2)).as(Void*) @context.resumable = 1 # actual stack top, not including guard pages and reserved pages LibC.GetNativeSystemInfo(out system_info) stack_top = @stack_bottom - system_info.dwPageSize + stack_ptr -= 4 # shadow space (or home space) before return address stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` From 4331c52190ea7d9b84e0bd2656be2e349602eafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 12:03:26 +0100 Subject: [PATCH 49/51] Simplify `Call.new` convenience overloads (#15427) --- src/compiler/crystal/syntax/ast.cr | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index fb443e2e6777..70db1bdda108 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -656,30 +656,22 @@ module Crystal property? args_in_brackets = false property? has_parentheses = false - def initialize(@obj, @name, @args = [] of ASTNode, @block = nil, @block_arg = nil, @named_args = nil, @global : Bool = false) + def initialize(@obj, @name, @args : Array(ASTNode) = [] of ASTNode, @block = nil, @block_arg = nil, @named_args = nil, @global : Bool = false) if block = @block block.call = self end end - def self.new(obj, name, arg : ASTNode, global = false) - new obj, name, [arg] of ASTNode, global: global + def self.new(obj, name, *args : ASTNode, global = false) + {% if compare_versions(Crystal::VERSION, "1.5.0") > 0 %} + new obj, name, [*args] of ASTNode, global: global + {% else %} + new obj, name, args.to_a(&.as(ASTNode)), global: global + {% end %} end - def self.new(obj, name, arg1 : ASTNode, arg2 : ASTNode) - new obj, name, [arg1, arg2] of ASTNode - end - - def self.new(obj, name, arg1 : ASTNode, arg2 : ASTNode, arg3 : ASTNode) - new obj, name, [arg1, arg2, arg3] of ASTNode - end - - def self.global(name, arg : ASTNode) - new nil, name, [arg] of ASTNode, global: true - end - - def self.global(name, arg1 : ASTNode, arg2 : ASTNode) - new nil, name, [arg1, arg2] of ASTNode, global: true + def self.global(name, *args : ASTNode) + new nil, name, *args, global: true end def name_size From 6e80a8a3d98fc6f987b5909e25abf0e7b738dfe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 15:53:16 +0100 Subject: [PATCH 50/51] Extract `regex_value` helper for macro methods (#15435) --- src/compiler/crystal/macros/methods.cr | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 24992816c8f6..5d617a62f73a 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -655,12 +655,7 @@ module Crystal interpret_check_args do |arg| case arg when RegexLiteral - arg_value = arg.value - if arg_value.is_a?(StringLiteral) - regex = Regex.new(arg_value.value, arg.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(arg) BoolLiteral.new(!!(@value =~ regex)) else BoolLiteral.new(false) @@ -735,12 +730,7 @@ module Crystal raise "first argument to StringLiteral#gsub must be a regex, not #{first.class_desc}" unless first.is_a?(RegexLiteral) raise "second argument to StringLiteral#gsub must be a string, not #{second.class_desc}" unless second.is_a?(StringLiteral) - regex_value = first.value - if regex_value.is_a?(StringLiteral) - regex = Regex.new(regex_value.value, first.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(first) StringLiteral.new(value.gsub(regex, second.value)) end @@ -764,12 +754,7 @@ module Crystal raise "StringLiteral#scan expects a regex, not #{arg.class_desc}" end - regex_value = arg.value - if regex_value.is_a?(StringLiteral) - regex = Regex.new(regex_value.value, arg.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(arg) matches = ArrayLiteral.new( of: Generic.new( @@ -912,6 +897,15 @@ module Crystal def to_macro_id @value end + + def regex_value(arg) + regex_value = arg.value + if regex_value.is_a?(StringLiteral) + Regex.new(regex_value.value, arg.options) + else + raise "regex interpolations not yet allowed in macros" + end + end end class StringInterpolation From cb7782d0f9aa53d68210dfcaa38179988136a7a3 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 11 Feb 2025 12:55:30 +0100 Subject: [PATCH 51/51] Initialize `Fiber` with an explicit stack (#15409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doesn't change the public API (the stack is still taken from the current scheduler's stack pool), but introduces an undocumented initializer that takes the stack and stack_bottom pointers. This will allow a few scenarios, mostly for RFC 2: - start fibers with a fake stack when we don't need to run the fibers, for example during specs that only need fiber objects (and would leak memory since we only release stacks after the fiber has run); - during a cross execution context spawn, we can pick a stack from the destination context instead of the current context (so we can recycle stacks from fibers that terminated in the desination context). * Add Fiber::Stack Holds the stack limits and whether the stack can be reused (i.e. released back into Fiber::StackPool). Also abstracts accessing the first addressable pointer with 16-bytes alignment (as required by most architectures) to pass to `makecontext`. Co-authored-by: Johannes Müller --- src/compiler/crystal/interpreter/context.cr | 4 +-- src/crystal/scheduler.cr | 2 +- src/crystal/system/unix/signal.cr | 4 +-- src/fiber.cr | 36 +++++++++++---------- src/fiber/context/aarch64-microsoft.cr | 8 ++--- src/fiber/context/x86_64-microsoft.cr | 8 ++--- src/fiber/stack.cr | 17 ++++++++++ src/fiber/stack_pool.cr | 19 +++++------ src/gc/boehm.cr | 2 +- 9 files changed, 60 insertions(+), 40 deletions(-) create mode 100644 src/fiber/stack.cr diff --git a/src/compiler/crystal/interpreter/context.cr b/src/compiler/crystal/interpreter/context.cr index 987781c4aefb..06a67c456370 100644 --- a/src/compiler/crystal/interpreter/context.cr +++ b/src/compiler/crystal/interpreter/context.cr @@ -106,10 +106,10 @@ class Crystal::Repl::Context # Once the block returns, the stack is returned to the pool. # The stack is not cleared after or before it's used. def checkout_stack(& : UInt8* -> _) - stack, _ = @stack_pool.checkout + stack = @stack_pool.checkout begin - yield stack.as(UInt8*) + yield stack.pointer.as(UInt8*) ensure @stack_pool.release(stack) end diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index efee6b3c06f1..51494fa2944b 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -124,7 +124,7 @@ class Crystal::Scheduler {% elsif flag?(:interpreted) %} # No need to change the stack bottom! {% else %} - GC.set_stackbottom(fiber.@stack_bottom) + GC.set_stackbottom(fiber.@stack.bottom) {% end %} current, @thread.current_fiber = @thread.current_fiber, fiber diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 12804ea00267..6c992478db5f 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -183,8 +183,8 @@ module Crystal::System::Signal is_stack_overflow = begin - stack_top = Pointer(Void).new(::Fiber.current.@stack.address - 4096) - stack_bottom = ::Fiber.current.@stack_bottom + stack_top = ::Fiber.current.@stack.pointer - 4096 + stack_bottom = ::Fiber.current.@stack.bottom stack_top <= addr < stack_bottom rescue e Crystal::System.print_error "Error while trying to determine if a stack overflow has occurred. Probable memory corruption\n" diff --git a/src/fiber.cr b/src/fiber.cr index 60162e5872a5..a7282047e165 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -1,6 +1,7 @@ require "crystal/system/thread_linked_list" require "crystal/print_buffered" require "./fiber/context" +require "./fiber/stack" # :nodoc: @[NoInline] @@ -56,12 +57,11 @@ class Fiber end @context : Context - @stack : Void* + @stack : Stack @resume_event : Crystal::EventLoop::Event? @timeout_event : Crystal::EventLoop::Event? # :nodoc: property timeout_select_action : Channel::TimeoutAction? - protected property stack_bottom : Void* # The name of the fiber, used as internal reference. property name : String? @@ -97,31 +97,30 @@ class Fiber # When the fiber is executed, it runs *proc* in its context. # # *name* is an optional and used only as an internal reference. - def initialize(@name : String? = nil, &@proc : ->) - @context = Context.new - @stack, @stack_bottom = + def self.new(name : String? = nil, &proc : ->) + stack = {% if flag?(:interpreted) %} - {Pointer(Void).null, Pointer(Void).null} + # the interpreter is managing the stacks + Stack.new(Pointer(Void).null, Pointer(Void).null) {% else %} Crystal::Scheduler.stack_pool.checkout {% end %} + new(name, stack, &proc) + end - fiber_main = ->(f : Fiber) { f.run } - - # point to first addressable pointer on the stack (@stack_bottom points past - # the stack because the stack grows down): - stack_ptr = @stack_bottom - sizeof(Void*) - - # align the stack pointer to 16 bytes: - stack_ptr = Pointer(Void*).new(stack_ptr.address & ~0x0f_u64) + # :nodoc: + def initialize(@name : String?, @stack : Stack, &@proc : ->) + @context = Context.new + fiber_main = ->(f : Fiber) { f.run } + stack_ptr = @stack.first_addressable_pointer makecontext(stack_ptr, fiber_main) Fiber.fibers.push(self) end # :nodoc: - def initialize(@stack : Void*, thread) + def initialize(stack : Void*, thread) @proc = Proc(Void).new { } # TODO: should creating a new context for the main fiber also be platform specific? @@ -133,7 +132,10 @@ class Fiber {% else %} Context.new(_fiber_get_stack_top) {% end %} - thread.gc_thread_handler, @stack_bottom = GC.current_thread_stack_bottom + + thread.gc_thread_handler, stack_bottom = GC.current_thread_stack_bottom + @stack = Stack.new(stack, stack_bottom) + @name = "main" {% if flag?(:preview_mt) %} @current_thread.set(thread) {% end %} Fiber.fibers.push(self) @@ -317,7 +319,7 @@ class Fiber # :nodoc: def push_gc_roots : Nil # Push the used section of the stack - GC.push_stack @context.stack_top, @stack_bottom + GC.push_stack @context.stack_top, @stack.bottom end {% if flag?(:preview_mt) %} diff --git a/src/fiber/context/aarch64-microsoft.cr b/src/fiber/context/aarch64-microsoft.cr index b2fa76580418..b9e86dfbc6cf 100644 --- a/src/fiber/context/aarch64-microsoft.cr +++ b/src/fiber/context/aarch64-microsoft.cr @@ -13,16 +13,16 @@ class Fiber # actual stack top, not including guard pages and reserved pages LibC.GetNativeSystemInfo(out system_info) - stack_top = @stack_bottom - system_info.dwPageSize + stack_top = @stack.bottom - system_info.dwPageSize stack_ptr[-4] = self.as(Void*) # x0 (r0): puts `self` as first argument for `fiber_main` stack_ptr[-16] = fiber_main.pointer # x30 (lr): initial `resume` will `ret` to this address # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits - stack_ptr[-3] = @stack # [x18, #0x1478]: Win32 DeallocationStack - stack_ptr[-2] = stack_top # [x18, #16]: Stack Limit - stack_ptr[-1] = @stack_bottom # [x18, #8]: Stack Base + stack_ptr[-3] = @stack.pointer # [x18, #0x1478]: Win32 DeallocationStack + stack_ptr[-2] = stack_top # [x18, #16]: Stack Limit + stack_ptr[-1] = @stack.bottom # [x18, #8]: Stack Base end # :nodoc: diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index a1b9fa281074..3a405e24d1f9 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -12,7 +12,7 @@ class Fiber # actual stack top, not including guard pages and reserved pages LibC.GetNativeSystemInfo(out system_info) - stack_top = @stack_bottom - system_info.dwPageSize + stack_top = @stack.bottom - system_info.dwPageSize stack_ptr -= 4 # shadow space (or home space) before return address stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address @@ -20,9 +20,9 @@ class Fiber # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits - stack_ptr[-2] = @stack # %gs:0x1478: Win32 DeallocationStack - stack_ptr[-3] = stack_top # %gs:0x10: Stack Limit - stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base + stack_ptr[-2] = @stack.pointer # %gs:0x1478: Win32 DeallocationStack + stack_ptr[-3] = stack_top # %gs:0x10: Stack Limit + stack_ptr[-4] = @stack.bottom # %gs:0x08: Stack Base end # :nodoc: diff --git a/src/fiber/stack.cr b/src/fiber/stack.cr new file mode 100644 index 000000000000..9666b506db0c --- /dev/null +++ b/src/fiber/stack.cr @@ -0,0 +1,17 @@ +class Fiber + # :nodoc: + struct Stack + getter pointer : Void* + getter bottom : Void* + getter? reusable : Bool + + def initialize(@pointer, @bottom, *, @reusable = false) + end + + def first_addressable_pointer : Void** + ptr = @bottom # stacks grow down + ptr -= sizeof(Void*) # point to first addressable pointer + Pointer(Void*).new(ptr.address & ~15_u64) # align to 16 bytes + end + end +end diff --git a/src/fiber/stack_pool.cr b/src/fiber/stack_pool.cr index 8f809335f46c..04954de40a94 100644 --- a/src/fiber/stack_pool.cr +++ b/src/fiber/stack_pool.cr @@ -12,12 +12,12 @@ class Fiber # Interpreter stacks grow upwards (pushing values increases the stack # pointer value) rather than downwards, so *protect* must be false. def initialize(@protect : Bool = true) - @deque = Deque(Void*).new + @deque = Deque(Stack).new end def finalize @deque.each do |stack| - Crystal::System::Fiber.free_stack(stack, STACK_SIZE) + Crystal::System::Fiber.free_stack(stack.pointer, STACK_SIZE) end end @@ -26,7 +26,7 @@ class Fiber def collect(count = lazy_size // 2) : Nil count.times do if stack = @deque.shift? - Crystal::System::Fiber.free_stack(stack, STACK_SIZE) + Crystal::System::Fiber.free_stack(stack.pointer, STACK_SIZE) else return end @@ -41,18 +41,19 @@ class Fiber end # Removes a stack from the bottom of the pool, or allocates a new one. - def checkout : {Void*, Void*} + def checkout : Stack if stack = @deque.pop? - Crystal::System::Fiber.reset_stack(stack, STACK_SIZE, @protect) + Crystal::System::Fiber.reset_stack(stack.pointer, STACK_SIZE, @protect) + stack else - stack = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + pointer = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + Stack.new(pointer, pointer + STACK_SIZE, reusable: true) end - {stack, stack + STACK_SIZE} end # Appends a stack to the bottom of the pool. - def release(stack) : Nil - @deque.push(stack) + def release(stack : Stack) : Nil + @deque.push(stack) if stack.reusable? end # Returns the approximated size of the pool. It may be equal or slightly diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 327b3d50409f..3a7a63d68153 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -223,7 +223,7 @@ module GC {% if flag?(:preview_mt) %} Thread.unsafe_each do |thread| if fiber = thread.current_fiber? - GC.set_stackbottom(thread.gc_thread_handler, fiber.@stack_bottom) + GC.set_stackbottom(thread.gc_thread_handler, fiber.@stack.bottom) end end {% end %}