Arm64: Implement region write barriers

src/coreclr/vm/arm64/asmhelpers.S

@@ -194,8 +194,9 @@ LEAF_END ThePreStubPatch, _TEXT
 LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT
     PROLOG_SAVE_REG_PAIR_INDEXED   fp, lr, -16
 
-    // x0-x7, x10 will contain intended new state
+    // x0-x7, x10-x11, x13-x14 will contain intended new state
     // x8 will preserve skipEphemeralCheck
+    // x9 will preserve writeableOffset
     // x12 will be used for pointers
 
     mov x8, x0
@@ -231,12 +232,28 @@ LOCAL_LABEL(EphemeralCheckEnabled):
     PREPARE_EXTERNAL_VAR g_highest_address, x12
     ldr  x6, [x12]
 
+    PREPARE_EXTERNAL_VAR g_region_to_generation_table, x12
+    ldr  x7, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_shr, x12
+    ldr  w10, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12
+    ldr  w11, [x12]
+
+    // Only allow bitwise write barriers if LSE atomics are present
+    PREPARE_EXTERNAL_VAR g_arm64_atomics_present, x12
+    ldr  w13, [x12]
+    cbnz w13, LOCAL_LABEL(AtomicsPresent)
+    movz w11, 0
+LOCAL_LABEL(AtomicsPresent):
+
 #ifdef WRITE_BARRIER_CHECK
     PREPARE_EXTERNAL_VAR g_GCShadow, x12
-    ldr  x7, [x12]
+    ldr  x13, [x12]
 
     PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12
-    ldr  x10, [x12]
+    ldr  x14, [x12]
 #endif
 
     // Update wbs state
@@ -247,12 +264,12 @@ LOCAL_LABEL(EphemeralCheckEnabled):
     stp  x0, x1, [x12], 16
     stp  x2, x3, [x12], 16
     stp  x4, x5, [x12], 16
-    str  x6, [x12], 8
+    stp  x6, x7, [x12], 16
+    stp  w10, w11, [x12], 8
 #ifdef WRITE_BARRIER_CHECK
-    stp  x7, x10, [x12], 16
+    stp  x13, x14, [x12], 16
 #endif
 
-
     EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16
     EPILOG_RETURN
 LEAF_END JIT_UpdateWriteBarrierState

src/coreclr/vm/arm64/patchedcode.S

@@ -142,33 +142,79 @@ LOCAL_LABEL(ShadowUpdateEnd):
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
     // Update the write watch table if necessary
     ldr  x12, LOCAL_LABEL(wbs_sw_ww_table)
-    cbz  x12, LOCAL_LABEL(CheckCardTable)
+    cbz  x12, LOCAL_LABEL(CheckCardTableBounds)
     add  x12, x12, x14, lsr #0xc  // SoftwareWriteWatch::AddressToTableByteIndexShift
     ldrb w17, [x12]
-    cbnz x17, LOCAL_LABEL(CheckCardTable)
+    cbnz x17, LOCAL_LABEL(CheckCardTableBounds)
     mov  w17, #0xFF
     strb w17, [x12]
 #endif
 
-LOCAL_LABEL(CheckCardTable):
-    // Branch to Exit if the reference is not in the Gen0 heap
+LOCAL_LABEL(CheckCardTableBounds):
+    // Branch to Exit if the reference is not in the heap
     ldr  x12, LOCAL_LABEL(wbs_ephemeral_low)
     ldr  x17, LOCAL_LABEL(wbs_ephemeral_high)
     cmp  x15, x12
     ccmp x15, x17, #0x2, hs
     bhs  LOCAL_LABEL(Exit)
 
+    // Region Checks
+
+    // Check if using regions
+    ldr  x17, LOCAL_LABEL(wbs_region_to_generation_table)
+    cbz  x17, LOCAL_LABEL(CheckCardTable)
+
+    // Calculate region locations
+    ldr  w12, LOCAL_LABEL(wbs_region_shr)
+    lsr  x15, x15, x12
+    add  x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table
+    lsr  x12, x14, x12
+    add  x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table
+
+    // Check whether the region we're storing into is gen 0 - nothing to do in this case
+    ldrb w12, [x12]
+    cbz  w12, LOCAL_LABEL(Exit)
+
+    // Check this is going from old to young
+    ldrb w15, [x15]
+    cmp  w15, w12
+    bhs  LOCAL_LABEL(Exit)
+
+    // Bitwise write barriers only
+    ldr  w17, LOCAL_LABEL(wbs_region_use_bitwise_write_barrier)
+    cbz  w17, LOCAL_LABEL(CheckCardTable)
+
     // Check if we need to update the card table
+    lsr w17, w14, 8
+    and w17, w17, 7
+    movz w15, 1
+    lsl w17, w15, w17  // w17 = 1 << (RHS >> 8 && 7)
     ldr  x12, LOCAL_LABEL(wbs_card_table)
     add  x15, x12, x14, lsr #11
-    ldrb w12, [x15]
+    ldrb w12, [x15]  // w12 = [(RHS >> 11) + g_card_table]
+    tst  w12, w17
+    bne  LOCAL_LABEL(Exit)
+
+    // Atomically update the card table
+    // Requires LSE, but the code is only compiled for 8.0
+    .word 0x383131FF // stsetb w17, [x15]
+    b  LOCAL_LABEL(CheckCardBundleTable)
+
+    // End of Region Checks
+
+LOCAL_LABEL(CheckCardTable):
+    // Check if we need to update the card table
+    ldr  x12, LOCAL_LABEL(wbs_card_table)
+    add  x15, x12, x14, lsr #11
+    ldrb w12, [x15]  // w12 = [(RHS >> 11) + g_card_table]
     cmp  x12, 0xFF
     beq  LOCAL_LABEL(Exit)
 
     // Update the card table
     mov  x12, 0xFF
     strb w12, [x15]
 
+LOCAL_LABEL(CheckCardBundleTable):
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
     // Check if we need to update the card bundle table
     ldr  x12, LOCAL_LABEL(wbs_card_bundle_table)
@@ -208,6 +254,12 @@ LOCAL_LABEL(wbs_lowest_address):
     .quad 0
 LOCAL_LABEL(wbs_highest_address):
     .quad 0
+LOCAL_LABEL(wbs_region_to_generation_table):
+    .quad 0
+LOCAL_LABEL(wbs_region_shr):
+    .word 0
+LOCAL_LABEL(wbs_region_use_bitwise_write_barrier):
+    .word 0
 #ifdef WRITE_BARRIER_CHECK
 LOCAL_LABEL(wbs_GCShadow):
     .quad 0