squash

src/gc-debug.c

@@ -198,6 +198,7 @@ static void restore(void)
 
 static void gc_verify_track(jl_ptls_t ptls)
 {
+#ifdef DFS_MARK
     do {
         jl_gc_markqueue_t mq;
         mq.current = mq.start = (ptls->mark_queue).start;
@@ -242,10 +243,12 @@ static void gc_verify_track(jl_ptls_t ptls)
         }
         restore();
     } while(lostval != NULL);
+#endif
 }
 
 void gc_verify(jl_ptls_t ptls)
 {
+#ifdef DFS_MARK
     jl_gc_markqueue_t mq;
     mq.current = mq.start = (ptls->mark_queue).start;
     mq.end = (ptls->mark_queue).end;
@@ -285,6 +288,7 @@ void gc_verify(jl_ptls_t ptls)
     jl_gc_debug_print_status();
     jl_gc_debug_critical_error();
     abort();
+#endif
 }
 #endif
 
@@ -1272,6 +1276,7 @@ int gc_slot_to_arrayidx(void *obj, void *_slot)
 // `offset` will be added to `mq->current` for convenience in the debugger.
 NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_markqueue_t *mq, int offset)
 {
+#ifdef DFS_MARK
     jl_jmp_buf *old_buf = jl_get_safe_restore();
     jl_jmp_buf buf;
     jl_set_safe_restore(&buf);
@@ -1290,6 +1295,7 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_markqueue_t *mq, int off
         jl_((void*)(jl_datatype_t *)(o->header & ~(uintptr_t)0xf));
     }
     jl_set_safe_restore(old_buf);
+#endif
 }
 
 static int gc_logging_enabled = 0;

src/gc.c

@@ -2888,8 +2888,26 @@ void jl_init_thread_heap(jl_ptls_t ptls)
     // Initialize GC mark-queue
     size_t init_size = (1 << 17);
     jl_gc_markqueue_t *mq = &ptls->mark_queue;
-    mq->current = mq->start = (jl_value_t**)malloc_s(init_size * sizeof(jl_value_t*));
+#if defined(PREFETCH_MARK)
+    // Initialize prefetch buffer
+    jl_gc_prefetch_buf_t *pf_buf = &mq->prefetch_buf;
+    pf_buf->start = (jl_value_t **)malloc_s(PF_SIZE * sizeof(jl_value_t *));
+    pf_buf->top = 0;
+    pf_buf->bottom = 0;
+    pf_buf->size = PF_SIZE;
+    // Initialize mark-stack
+    jl_gc_markstack_t *ms = &mq->mark_stack;
+    ms->current = ms->start = (jl_value_t **)malloc_s(init_size * sizeof(jl_value_t *));
+    ms->end = ms->start + init_size;
+#elif defined(DFS_MARK)
+    mq->start = (jl_value_t **)malloc_s(init_size * sizeof(jl_value_t *));
+    mq->current = mq->start;
     mq->end = mq->start + init_size;
+#else
+    mq->start = (jl_value_t **)malloc_s(init_size * sizeof(jl_value_t *));
+    mq->top = mq->bottom = 0;
+    mq->capacity = init_size;
+#endif
 
     memset(&ptls->gc_num, 0, sizeof(ptls->gc_num));
     assert(gc_num.interval == default_collect_interval);

src/gc.h

@@ -84,32 +84,118 @@ typedef struct {
 // Double the mark queue
 static NOINLINE void gc_markqueue_resize(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT
 {
+#if defined(PREFETCH_MARK)
+    jl_gc_markstack_t *ms = &mq->mark_stack;
+    jl_value_t **old_start = ms->start;
+    size_t old_queue_size = (ms->end - ms->start);
+    size_t offset = (ms->current - old_start);
+    ms->start = (jl_value_t**)realloc_s(old_start, 2 * old_queue_size * sizeof(jl_value_t*));
+    ms->current = (ms->start + offset);
+    ms->end = (ms->start + 2 * old_queue_size);
+#elif defined(DFS_MARK)
     jl_value_t **old_start = mq->start;
     size_t old_queue_size = (mq->end - mq->start);
     size_t offset = (mq->current - old_start);
     mq->start =
         (jl_value_t **)realloc_s(old_start, 2 * old_queue_size * sizeof(jl_value_t *));
     mq->current = (mq->start + offset);
     mq->end = (mq->start + 2 * old_queue_size);
+#else
+    jl_value_t **old_start = mq->start;
+    size_t old_capacity = mq->capacity;
+    mq->start = (jl_value_t **)malloc(2 * old_capacity * sizeof(jl_value_t *));
+    // Copy elements into new buffer
+    for (size_t i = mq->top; i < mq->bottom; i++)
+        mq->start[i % (2 * old_capacity)] = old_start[i % old_capacity];
+    free(old_start);
+    mq->capacity = 2 * old_capacity;
+#endif
 }
 
+#define PF_MIN (1 << 6)
+#define PF_SIZE (1 << 8)
+
 // Push a work item to the queue
-STATIC_INLINE void gc_markqueue_push(jl_gc_markqueue_t *mq, jl_value_t *obj) JL_NOTSAFEPOINT
+STATIC_INLINE void gc_markqueue_push(jl_gc_markqueue_t *mq,
+                                     jl_value_t *obj) JL_NOTSAFEPOINT
 {
+#if defined(PREFETCH_MARK)
+    jl_gc_prefetch_buf_t *pf_buf = &mq->prefetch_buf;
+    jl_gc_markstack_t *ms = &mq->mark_stack;
+    // Prefetch buffer overflowed: push to mark-stack
+    if (__likely(pf_buf->bottom - pf_buf->top >= pf_buf->size)) {
+        // Mark-stack overflowed: resize it
+        if (__unlikely(ms->current == ms->end))
+            gc_markqueue_resize(mq);
+        *ms->current = obj;
+        ms->current++;
+    }
+    else {
+        // There is still space in the FIFO buffer: push it there
+        pf_buf->start[pf_buf->bottom % pf_buf->size] = obj;
+        pf_buf->bottom++;
+    }
+#elif defined(DFS_MARK)
     if (__unlikely(mq->current == mq->end))
         gc_markqueue_resize(mq);
     *mq->current = obj;
     mq->current++;
+#else
+    // Mark-stack overflowed: resize it
+    if (__unlikely(mq->bottom - mq->top >= mq->capacity))
+        gc_markqueue_resize(mq);
+    mq->start[mq->bottom % mq->capacity] = obj;
+    mq->bottom++;
+#endif
 }
 
+#if defined(_COMPILER_GCC_) && defined(_CPU_X86_)
+#define jl_prefetch(p) __builtin_prefetch((p), 1, 3)
+#else
+#define jl_prefetch(p)
+#endif
+
 // Pop from the mark queue
 STATIC_INLINE jl_value_t *gc_markqueue_pop(jl_gc_markqueue_t *mq)
 {
+#if defined(PREFETCH_MARK)
+    jl_gc_prefetch_buf_t *pf_buf = &mq->prefetch_buf;
+    jl_gc_markstack_t *ms = &mq->mark_stack;
+    jl_value_t *obj = NULL;
+    // FIFO buffer is nearly empty and there is element in mark-stack: pop
+    // element from stack
+    if (pf_buf->bottom - pf_buf->top <= PF_MIN && ms->current != ms->start) {
+        ms->current--;
+        obj = *ms->current;
+    }
+    // enough elements in FIFO buffer
+    else if (pf_buf->bottom - pf_buf->top > 0) {
+        // take element from FIFO buffer
+        obj = pf_buf->start[pf_buf->top % pf_buf->size];
+        pf_buf->top++;
+        // if there is any element in the stack, pop it, prefetch and insert into FIFO buffer
+        if (ms->current != ms->start) {
+            ms->current--;
+            jl_value_t *to_prefetch = *ms->current;
+            jl_prefetch(to_prefetch);
+            pf_buf->start[pf_buf->bottom % pf_buf->size] = to_prefetch;
+            pf_buf->bottom++;
+        }
+    }
+    return obj;
+#elif defined(DFS_MARK)
     if (mq->current == mq->start)
         return NULL;
     mq->current--;
     jl_value_t *obj = *mq->current;
     return obj;
+#else
+    if (mq->bottom == mq->top)
+        return NULL;
+    jl_value_t *obj = mq->start[mq->top % mq->capacity];
+    mq->top++;
+    return obj;
+#endif
 }
 
 // layout for big (>2k) objects

src/julia_threads.h

@@ -168,14 +168,43 @@ typedef struct {
     arraylist_t free_stacks[JL_N_STACK_POOLS];
 } jl_thread_heap_t;
 
-// Cache of thread local change to global metadata during GC
-// This is sync'd after marking.
-typedef union _jl_gc_mark_data jl_gc_mark_data_t;
+// #define PREFETCH_MARK
+
+#ifndef PREFETCH_MARK
+#define DFS_MARK
+#endif
+
+#ifdef PREFETCH_MARK
+
+typedef struct {
+    struct _jl_value_t **start;
+    size_t top;
+    size_t bottom;
+    size_t size;
+} jl_gc_prefetch_buf_t;
+
+typedef struct {
+    struct _jl_value_t **start;
+    struct _jl_value_t **current;
+    struct _jl_value_t **end;
+} jl_gc_markstack_t;
+
+#endif
 
 typedef struct {
+#if defined(PREFETCH_MARK)
+    jl_gc_prefetch_buf_t prefetch_buf;
+    jl_gc_markstack_t mark_stack;
+#elif defined(DFS_MARK)
     struct _jl_value_t **start;
     struct _jl_value_t **current;
     struct _jl_value_t **end;
+#else
+    struct _jl_value_t **start;
+    size_t top;
+    size_t bottom;
+    size_t capacity;
+#endif
 } jl_gc_markqueue_t;
 
 typedef struct {