smaller Task object

- move locks array to thread-local state
- move tid,prio to occupy existing alignment padding

src/julia.h

@@ -1782,6 +1782,11 @@ typedef struct _jl_task_t {
     uint8_t sticky; // record whether this Task can be migrated to a new thread
 
 // hidden state:
+    // id of owning thread - does not need to be defined until the task runs
+    int16_t tid;
+    // multiqueue priority
+    int16_t prio;
+
     jl_ucontext_t ctx; // saved thread state
     void *stkbuf; // malloc'd memory (either copybuf or stack)
     size_t bufsz; // actual sizeof stkbuf
@@ -1797,13 +1802,6 @@ typedef struct _jl_task_t {
     // current world age
     size_t world_age;
 
-    // id of owning thread
-    // does not need to be defined until the task runs
-    int16_t tid;
-    /* for the multiqueue */
-    int16_t prio;
-    // This is statically initialized when the task is not holding any locks
-    arraylist_t locks;
     jl_timing_block_t *timing_stack;
 } jl_task_t;
 

src/julia_threads.h

@@ -226,6 +226,9 @@ struct _jl_tls_states_t {
     // Access via jl_exception_occurred().
     struct _jl_value_t *previous_exception;
 
+    // currently-held locks, to be released when an exception is thrown
+    small_arraylist_t locks;
+
     JULIA_DEBUG_SLEEPWAKE(
         uint64_t uv_run_enter;
         uint64_t uv_run_leave;

src/locks.h

@@ -54,13 +54,10 @@ static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT
 static inline void jl_lock_frame_push(jl_mutex_t *lock)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
-    // For early bootstrap
-    if (__unlikely(!ptls->current_task))
-        return;
-    arraylist_t *locks = &ptls->current_task->locks;
-    size_t len = locks->len;
+    small_arraylist_t *locks = &ptls->locks;
+    uint32_t len = locks->len;
     if (__unlikely(len >= locks->max)) {
-        arraylist_grow(locks, 1);
+        small_arraylist_grow(locks, 1);
     }
     else {
         locks->len = len + 1;
@@ -70,9 +67,7 @@ static inline void jl_lock_frame_push(jl_mutex_t *lock)
 static inline void jl_lock_frame_pop(void)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
-    if (__likely(ptls->current_task)) {
-        ptls->current_task->locks.len--;
-    }
+    ptls->locks.len--;
 }
 
 #define JL_SIGATOMIC_BEGIN() do {               \

src/rtutils.c

@@ -213,7 +213,7 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh)
     eh->prev = current_task->eh;
     eh->gcstack = ptls->pgcstack;
     eh->gc_state = ptls->gc_state;
-    eh->locks_len = current_task->locks.len;
+    eh->locks_len = ptls->locks.len;
     eh->defer_signal = ptls->defer_signal;
     eh->finalizers_inhibited = ptls->finalizers_inhibited;
     eh->world_age = ptls->world_age;
@@ -245,7 +245,7 @@ JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh)
     int8_t old_gc_state = ptls->gc_state;
     current_task->eh = eh->prev;
     ptls->pgcstack = eh->gcstack;
-    arraylist_t *locks = &current_task->locks;
+    small_arraylist_t *locks = &ptls->locks;
     if (locks->len > eh->locks_len) {
         for (size_t i = locks->len;i > eh->locks_len;i--)
             jl_mutex_unlock_nogc((jl_mutex_t*)locks->items[i - 1]);

src/support/arraylist.c

@@ -78,6 +78,73 @@ void *arraylist_pop(arraylist_t *a)
     return p;
 }
 
+// small arraylist
+
+small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size)
+{
+    a->len = 0;
+    if (size <= SMALL_AL_N_INLINE) {
+        a->items = &a->_space[0];
+        a->max = SMALL_AL_N_INLINE;
+    }
+    else {
+        a->items = (void**)LLT_ALLOC(size*sizeof(void*));
+        if (a->items == NULL) return NULL;
+        a->max = size;
+    }
+    return a;
+}
+
+void small_arraylist_free(small_arraylist_t *a)
+{
+    if (a->items != &a->_space[0])
+        LLT_FREE(a->items);
+    a->len = 0;
+    a->max = SMALL_AL_N_INLINE;
+    a->items = &a->_space[0];
+}
+
+void small_arraylist_grow(small_arraylist_t *a, uint32_t n)
+{
+    size_t len = a->len;
+    size_t newlen = len + n;
+    if (newlen > a->max) {
+        if (a->items == &a->_space[0]) {
+            void **p = (void**)LLT_ALLOC((a->len+n)*sizeof(void*));
+            if (p == NULL) return;
+            memcpy(p, a->items, len * sizeof(void*));
+            a->items = p;
+            a->max = newlen;
+        }
+        else {
+            size_t nm = a->max * 2;
+            if (nm == 0)
+                nm = 1;
+            while (newlen > nm)
+                nm *= 2;
+            void **p = (void**)LLT_REALLOC(a->items, nm * sizeof(void*));
+            if (p == NULL) return;
+            a->items = p;
+            a->max = nm;
+        }
+    }
+    a->len = newlen;
+}
+
+void small_arraylist_push(small_arraylist_t *a, void *elt)
+{
+    small_arraylist_grow(a, 1);
+    a->items[a->len - 1] = elt;
+}
+
+void *small_arraylist_pop(small_arraylist_t *a)
+{
+    if (a->len == 0) return NULL;
+    void *p = a->items[--a->len];
+    a->items[a->len] = NULL;
+    return p;
+}
+
 #ifdef __cplusplus
 }
 #endif

src/support/arraylist.h

@@ -5,6 +5,8 @@
 
 #define AL_N_INLINE 29
 
+#define SMALL_AL_N_INLINE 6
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -25,6 +27,20 @@ void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT;
 void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT;
 void arraylist_grow(arraylist_t *a, size_t n) JL_NOTSAFEPOINT;
 
+typedef struct {
+    uint32_t len;
+    uint32_t max;
+    void **items;
+    void *_space[SMALL_AL_N_INLINE];
+} small_arraylist_t;
+
+small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size) JL_NOTSAFEPOINT;
+void small_arraylist_free(small_arraylist_t *a) JL_NOTSAFEPOINT;
+
+void small_arraylist_push(small_arraylist_t *a, void *elt) JL_NOTSAFEPOINT;
+void *small_arraylist_pop(small_arraylist_t *a) JL_NOTSAFEPOINT;
+void small_arraylist_grow(small_arraylist_t *a, uint32_t n) JL_NOTSAFEPOINT;
+
 #ifdef __cplusplus
 }
 #endif

src/task.c

@@ -269,13 +269,8 @@ static void ctx_switch(jl_ptls_t ptls)
     jl_task_t *t = *pt;
     assert(t != ptls->current_task);
     jl_task_t *lastt = ptls->current_task;
-    // If the current task is not holding any locks, free the locks list
-    // so that it can be GC'd without leaking memory
-    arraylist_t *locks = &lastt->locks;
-    if (locks->len == 0 && locks->items != locks->_space) {
-        arraylist_free(locks);
-        arraylist_new(locks, 0);
-    }
+    // none of these locks should be held across a task switch
+    assert(ptls->locks.len == 0);
 
     int killed = (lastt->state == done_sym || lastt->state == failed_sym);
     if (!t->started && !t->copy_stack) {
@@ -593,7 +588,6 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion
 #ifdef ENABLE_TIMINGS
     t->timing_stack = jl_root_timing;
 #endif
-    arraylist_new(&t->locks, 0);
 
 #if defined(JL_DEBUG_BUILD)
     if (!t->copy_stack)
@@ -1108,7 +1102,6 @@ void jl_init_root_task(void *stack_lo, void *stack_hi)
     ptls->current_task->excstack = NULL;
     ptls->current_task->tid = ptls->tid;
     ptls->current_task->sticky = 1;
-    arraylist_new(&ptls->current_task->locks, 0);
 
 #ifdef COPY_STACKS
     if (always_copy_stacks) {

src/threading.c

@@ -289,6 +289,7 @@ void jl_init_threadtls(int16_t tid)
 #ifdef _OS_WINDOWS_
     ptls->needs_resetstkoflw = 0;
 #endif
+    small_arraylist_new(&ptls->locks, 0);
     jl_init_thread_heap(ptls);
     jl_install_thread_signal_handler(ptls);
 

test/threads.jl

@@ -95,9 +95,11 @@ function Base.wait(idle::UvTestIdle)
     Base.lock(idle.cond)
     try
         idle.active = true
+        Base.iolock_end()
         wait(idle.cond)
     finally
         Base.unlock(idle.cond)
+        Base.iolock_begin()
         Base.unpreserve_handle(idle)
         Base.iolock_end()
     end