From cd68d0fed9dae5c5051421fafefa537b0a3223c7 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 27 Jul 2016 16:25:50 -0400 Subject: [PATCH] fix #13529, slowdown with large number of async `sleep` calls The problem was performance degradation of ObjectIdDict with many deleted items. The table needs to be rehashed after a large number of deletions. --- base/dict.jl | 27 ++++++++++++++++++++++----- src/dump.c | 2 +- src/julia_internal.h | 2 +- src/table.c | 12 ++++++------ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 00e56434d27459..7ca9c5aaf54d1b 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -247,7 +247,8 @@ push!(t::Associative, p::Pair, q::Pair, r::Pair...) = push!(push!(push!(t, p), q type ObjectIdDict <: Associative{Any,Any} ht::Vector{Any} - ObjectIdDict() = new(Vector{Any}(32)) + ndel::Int + ObjectIdDict() = new(Vector{Any}(32), 0) function ObjectIdDict(itr) d = ObjectIdDict() @@ -266,7 +267,16 @@ end similar(d::ObjectIdDict) = ObjectIdDict() +function rehash!(t::ObjectIdDict, newsz = length(t.ht)) + t.ht = ccall(:jl_idtable_rehash, Any, (Any, Csize_t), t.ht, newsz) + t +end + function setindex!(t::ObjectIdDict, v::ANY, k::ANY) + if t.ndel >= ((3*length(t.ht))>>2) + rehash!(t, length(t.ht)>>1) + t.ndel = 0 + end t.ht = ccall(:jl_eqtable_put, Array{Any,1}, (Any, Any, Any), t.ht, k, v) return t end @@ -274,8 +284,15 @@ end get(t::ObjectIdDict, key::ANY, default::ANY) = ccall(:jl_eqtable_get, Any, (Any, Any, Any), t.ht, key, default) -pop!(t::ObjectIdDict, key::ANY, default::ANY) = - ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, default) +function pop!(t::ObjectIdDict, key::ANY, default::ANY) + val = ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, secret_table_token) + if val !== secret_table_token + t.ndel += 1 + val + else + default + end +end function pop!(t::ObjectIdDict, key::ANY) val = pop!(t, key, secret_table_token) @@ -283,11 +300,11 @@ function pop!(t::ObjectIdDict, key::ANY) end function delete!(t::ObjectIdDict, key::ANY) - ccall(:jl_eqtable_pop, Any, (Any, Any), t.ht, key) + pop!(t, key, secret_table_token) t end -empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t) +empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t.ndel = 0; t) _oidd_nextind(a, i) = reinterpret(Int,ccall(:jl_eqtable_nextind, Csize_t, (Any, Csize_t), a, i)) diff --git a/src/dump.c b/src/dump.c index b10909d27c4d8d..6e9ee562393744 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1814,7 +1814,7 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) case 1: { // rehash ObjectIdDict jl_array_t **a = (jl_array_t**)v; // Assume *a don't need a write barrier - jl_idtable_rehash(a, jl_array_len(*a)); + *a = jl_idtable_rehash(*a, jl_array_len(*a)); jl_gc_wb(v, *a); break; } diff --git a/src/julia_internal.h b/src/julia_internal.h index d967de1ff00cbe..2981f086d7e0f1 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -411,7 +411,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); int32_t jl_assign_functionID(/*llvm::Function*/void *function); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns -void jl_idtable_rehash(jl_array_t **pa, size_t newsz); +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); diff --git a/src/table.c b/src/table.c index 5bed49efbfe265..f94bac2f78faa3 100644 --- a/src/table.c +++ b/src/table.c @@ -10,13 +10,13 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key); -void jl_idtable_rehash(jl_array_t **pa, size_t newsz) +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz) { // Assume *pa don't need a write barrier // pa doesn't have to be a GC slot but *pa needs to be rooted - size_t sz = jl_array_len(*pa); + size_t sz = jl_array_len(a); size_t i; - void **ol = (void**)(*pa)->data; + void **ol = (void**)a->data; jl_array_t *newa = jl_alloc_vec_any(newsz); // keep the original array in the original slot since we need `ol` // to be valid in the loop below. @@ -25,16 +25,16 @@ void jl_idtable_rehash(jl_array_t **pa, size_t newsz) if (ol[i+1] != NULL) { (*jl_table_lookup_bp(&newa, ol[i])) = ol[i+1]; jl_gc_wb(newa, ol[i+1]); - // it is however necessary here because allocation + // it is however necessary here because allocation // can (and will) occur in a recursive call inside table_lookup_bp } } - *pa = newa; // we do not check the write barrier here // because pa always points to a C stack location // (see jl_eqtable_put and jl_finalize_deserializer) // it should be changed if this assumption no longer holds JL_GC_POP(); + return newa; } static void **jl_table_lookup_bp(jl_array_t **pa, void *key) @@ -81,7 +81,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) newsz = HT_N_INLINE; else newsz = sz<<2; - jl_idtable_rehash(pa, newsz); + *pa = jl_idtable_rehash(*pa, newsz); a = *pa; tab = (void**)a->data;