diff --git a/src/julia.h b/src/julia.h index ba16b165e792e..e04e754904e65 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1799,6 +1799,11 @@ typedef struct _jl_task_t { uint8_t sticky; // record whether this Task can be migrated to a new thread // hidden state: + // id of owning thread - does not need to be defined until the task runs + int16_t tid; + // multiqueue priority + int16_t prio; + jl_ucontext_t ctx; // saved thread state void *stkbuf; // malloc'd memory (either copybuf or stack) size_t bufsz; // actual sizeof stkbuf @@ -1818,13 +1823,6 @@ typedef struct _jl_task_t { // current world age size_t world_age; - // id of owning thread - // does not need to be defined until the task runs - int16_t tid; - /* for the multiqueue */ - int16_t prio; - // This is statically initialized when the task is not holding any locks - arraylist_t locks; jl_timing_block_t *timing_stack; } jl_task_t; diff --git a/src/julia_threads.h b/src/julia_threads.h index 6040a593300c1..616316c4f9231 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -226,6 +226,9 @@ struct _jl_tls_states_t { // Access via jl_exception_occurred(). struct _jl_value_t *previous_exception; + // currently-held locks, to be released when an exception is thrown + small_arraylist_t locks; + JULIA_DEBUG_SLEEPWAKE( uint64_t uv_run_enter; uint64_t uv_run_leave; diff --git a/src/locks.h b/src/locks.h index 0f011c5e0a7f0..b9d4d0a9d93a6 100644 --- a/src/locks.h +++ b/src/locks.h @@ -54,13 +54,10 @@ static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT static inline void jl_lock_frame_push(jl_mutex_t *lock) { jl_ptls_t ptls = jl_get_ptls_states(); - // For early bootstrap - if (__unlikely(!ptls->current_task)) - return; - arraylist_t *locks = &ptls->current_task->locks; - size_t len = locks->len; + small_arraylist_t *locks = &ptls->locks; + uint32_t len = locks->len; if (__unlikely(len >= locks->max)) { - arraylist_grow(locks, 1); + small_arraylist_grow(locks, 1); } else { locks->len = len + 1; @@ -70,9 +67,7 @@ static inline void jl_lock_frame_push(jl_mutex_t *lock) static inline void jl_lock_frame_pop(void) { jl_ptls_t ptls = jl_get_ptls_states(); - if (__likely(ptls->current_task)) { - ptls->current_task->locks.len--; - } + ptls->locks.len--; } #define JL_SIGATOMIC_BEGIN() do { \ diff --git a/src/rtutils.c b/src/rtutils.c index 4e6a8342c5241..a7ef36ee9f788 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -213,7 +213,7 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) eh->prev = current_task->eh; eh->gcstack = ptls->pgcstack; eh->gc_state = ptls->gc_state; - eh->locks_len = current_task->locks.len; + eh->locks_len = ptls->locks.len; eh->defer_signal = ptls->defer_signal; eh->finalizers_inhibited = ptls->finalizers_inhibited; eh->world_age = ptls->world_age; @@ -245,7 +245,7 @@ JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh) int8_t old_gc_state = ptls->gc_state; current_task->eh = eh->prev; ptls->pgcstack = eh->gcstack; - arraylist_t *locks = ¤t_task->locks; + small_arraylist_t *locks = &ptls->locks; if (locks->len > eh->locks_len) { for (size_t i = locks->len;i > eh->locks_len;i--) jl_mutex_unlock_nogc((jl_mutex_t*)locks->items[i - 1]); diff --git a/src/support/arraylist.c b/src/support/arraylist.c index bb8bf77bd1af4..343ee59ab6540 100644 --- a/src/support/arraylist.c +++ b/src/support/arraylist.c @@ -78,6 +78,73 @@ void *arraylist_pop(arraylist_t *a) return p; } +// small arraylist + +small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size) +{ + a->len = 0; + if (size <= SMALL_AL_N_INLINE) { + a->items = &a->_space[0]; + a->max = SMALL_AL_N_INLINE; + } + else { + a->items = (void**)LLT_ALLOC(size*sizeof(void*)); + if (a->items == NULL) return NULL; + a->max = size; + } + return a; +} + +void small_arraylist_free(small_arraylist_t *a) +{ + if (a->items != &a->_space[0]) + LLT_FREE(a->items); + a->len = 0; + a->max = SMALL_AL_N_INLINE; + a->items = &a->_space[0]; +} + +void small_arraylist_grow(small_arraylist_t *a, uint32_t n) +{ + size_t len = a->len; + size_t newlen = len + n; + if (newlen > a->max) { + if (a->items == &a->_space[0]) { + void **p = (void**)LLT_ALLOC((a->len+n)*sizeof(void*)); + if (p == NULL) return; + memcpy(p, a->items, len * sizeof(void*)); + a->items = p; + a->max = newlen; + } + else { + size_t nm = a->max * 2; + if (nm == 0) + nm = 1; + while (newlen > nm) + nm *= 2; + void **p = (void**)LLT_REALLOC(a->items, nm * sizeof(void*)); + if (p == NULL) return; + a->items = p; + a->max = nm; + } + } + a->len = newlen; +} + +void small_arraylist_push(small_arraylist_t *a, void *elt) +{ + small_arraylist_grow(a, 1); + a->items[a->len - 1] = elt; +} + +void *small_arraylist_pop(small_arraylist_t *a) +{ + if (a->len == 0) return NULL; + void *p = a->items[--a->len]; + a->items[a->len] = NULL; + return p; +} + #ifdef __cplusplus } #endif diff --git a/src/support/arraylist.h b/src/support/arraylist.h index 2bd5a87ff6d2b..f996fb397c6e0 100644 --- a/src/support/arraylist.h +++ b/src/support/arraylist.h @@ -5,6 +5,8 @@ #define AL_N_INLINE 29 +#define SMALL_AL_N_INLINE 6 + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +27,20 @@ void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT; void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT; void arraylist_grow(arraylist_t *a, size_t n) JL_NOTSAFEPOINT; +typedef struct { + uint32_t len; + uint32_t max; + void **items; + void *_space[SMALL_AL_N_INLINE]; +} small_arraylist_t; + +small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size) JL_NOTSAFEPOINT; +void small_arraylist_free(small_arraylist_t *a) JL_NOTSAFEPOINT; + +void small_arraylist_push(small_arraylist_t *a, void *elt) JL_NOTSAFEPOINT; +void *small_arraylist_pop(small_arraylist_t *a) JL_NOTSAFEPOINT; +void small_arraylist_grow(small_arraylist_t *a, uint32_t n) JL_NOTSAFEPOINT; + #ifdef __cplusplus } #endif diff --git a/src/task.c b/src/task.c index 8fa20462bb52a..088e221300b05 100644 --- a/src/task.c +++ b/src/task.c @@ -287,13 +287,8 @@ static void ctx_switch(jl_ptls_t ptls) jl_task_t *t = *pt; assert(t != ptls->current_task); jl_task_t *lastt = ptls->current_task; - // If the current task is not holding any locks, free the locks list - // so that it can be GC'd without leaking memory - arraylist_t *locks = &lastt->locks; - if (locks->len == 0 && locks->items != locks->_space) { - arraylist_free(locks); - arraylist_new(locks, 0); - } + // none of these locks should be held across a task switch + assert(ptls->locks.len == 0); int killed = (lastt->state == done_sym || lastt->state == failed_sym); if (!t->started && !t->copy_stack) { @@ -617,7 +612,6 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion #ifdef JL_TSAN_ENABLED t->tsan_state = tsan_create_fiber(); #endif - arraylist_new(&t->locks, 0); #if defined(JL_DEBUG_BUILD) if (!t->copy_stack) @@ -1146,7 +1140,6 @@ void jl_init_root_task(void *stack_lo, void *stack_hi) ptls->current_task->excstack = NULL; ptls->current_task->tid = ptls->tid; ptls->current_task->sticky = 1; - arraylist_new(&ptls->current_task->locks, 0); #ifdef JL_TSAN_ENABLED ptls->current_task->tsan_state = tsan_create_fiber(); diff --git a/src/threading.c b/src/threading.c index 702c7016295da..848733428d186 100644 --- a/src/threading.c +++ b/src/threading.c @@ -289,6 +289,7 @@ void jl_init_threadtls(int16_t tid) #ifdef _OS_WINDOWS_ ptls->needs_resetstkoflw = 0; #endif + small_arraylist_new(&ptls->locks, 0); jl_init_thread_heap(ptls); jl_install_thread_signal_handler(ptls); diff --git a/test/threads.jl b/test/threads.jl index 3a3751d21a232..0a23e2ee6f950 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -95,9 +95,11 @@ function Base.wait(idle::UvTestIdle) Base.lock(idle.cond) try idle.active = true + Base.iolock_end() wait(idle.cond) finally Base.unlock(idle.cond) + Base.iolock_begin() Base.unpreserve_handle(idle) Base.iolock_end() end