fixup: use intset for keys

JuliaLang · Mar 26, 2020 · be8c8f5 · be8c8f5
1 parent bf16baf
commit be8c8f5
Show file tree

Hide file tree

Showing 11 changed files with 289 additions and 264 deletions.
diff --git a/src/Makefile b/src/Makefile
@@ -40,7 +40,7 @@ FLAGS += -I$(LOCALBASE)/include
 endif
 
 RUNTIME_SRCS := \
-	jltypes gf typemap typeset ast builtins module interpreter symbol \
+	jltypes gf typemap smallintset ast builtins module interpreter symbol \
 	dlload sys init task array dump staticdata toplevel jl_uv datatype \
 	simplevector runtime_intrinsics precompile \
 	threading partr stackwalk gc gc-debug gc-pages gc-stacks method \

diff --git a/src/dump.c b/src/dump.c
@@ -851,7 +851,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li
         assert((jl_value_t*)mt != jl_nothing);
         external_mt = !module_in_worklist(mt->module);
         jl_serialize_value(s, m->specializations);
-        jl_serialize_value(s, m->linearspecializations);
+        jl_serialize_value(s, m->speckeyset);
         jl_serialize_value(s, (jl_value_t*)m->name);
         jl_serialize_value(s, (jl_value_t*)m->file);
         write_int32(s->s, m->line);
@@ -1166,13 +1166,6 @@ static int jl_collect_methcache_from_mod(jl_typemap_entry_t *ml, void *closure)
             if (callee != NULL)
                 collect_backedges(callee);
         }
-        specializations = m->linearspecializations;
-        l = jl_svec_len(specializations);
-        for (i = 0; i < l; i++) {
-            jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(specializations, i);
-            if (callee != NULL)
-                collect_backedges(callee);
-        }
     }
     return 1;
 }
@@ -1727,8 +1720,8 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_
     }
     m->specializations = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->specializations);
     jl_gc_wb(m, m->specializations);
-    m->linearspecializations = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->linearspecializations);
-    jl_gc_wb(m, m->linearspecializations);
+    m->speckeyset = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->speckeyset);
+    jl_gc_wb(m, m->speckeyset);
     m->name = (jl_sym_t*)jl_deserialize_value(s, NULL);
     jl_gc_wb(m, m->name);
     m->file = (jl_sym_t*)jl_deserialize_value(s, NULL);

diff --git a/src/gf.c b/src/gf.c
@@ -86,17 +86,20 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt)
 
 /// ----- Insertion logic for special entries ----- ///
 
-static uint_t speccache_hash(jl_value_t *ml)
+
+static uint_t speccache_hash(size_t idx, jl_svec_t *data)
 {
-    jl_value_t *sig = ((jl_method_instance_t*)ml)->specTypes;
+    jl_method_instance_t *ml = (jl_method_instance_t*)jl_svecref(data, idx);
+    jl_value_t *sig = ml->specTypes;
     if (jl_is_unionall(sig))
         sig = jl_unwrap_unionall(sig);
     return ((jl_datatype_t*)sig)->hash;
 }
 
-static int speccache_eq(jl_value_t *ml, const void *ty, uint_t hv)
+static int speccache_eq(size_t idx, const void *ty, jl_svec_t *data, uint_t hv)
 {
-    jl_value_t *sig = ((jl_method_instance_t*)ml)->specTypes;
+    jl_method_instance_t *ml = (jl_method_instance_t*)jl_svecref(data, idx);
+    jl_value_t *sig = ml->specTypes;
     if (ty == sig)
         return 1;
     uint_t h2 = ((jl_datatype_t*)(jl_is_unionall(sig) ? jl_unwrap_unionall(sig) : sig))->hash;
@@ -109,91 +112,93 @@ static int speccache_eq(jl_value_t *ml, const void *ty, uint_t hv)
 JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams)
 {
     uint_t hv = ((jl_datatype_t*)(jl_is_unionall(type) ? jl_unwrap_unionall(type) : type))->hash;
-    if (hv) {
-        jl_method_instance_t *mi = (jl_method_instance_t*)jl_typeset_lookup(&m->specializations, speccache_eq, type, hv, jl_nothing);
-        if (jl_is_method_instance(mi))
-            return mi;
-    }
-    else {
-        jl_svec_t *linearcache = jl_atomic_load_relaxed(&m->linearspecializations);
-        jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(linearcache);
-        size_t i, cl = jl_svec_len(linearcache);
-        for (i = 0; i < cl; i++) {
-            jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
-            JL_GC_PROMISE_ROOTED(mi);
-            if (!mi)
-                break;
-            if (jl_types_equal(mi->specTypes, type))
+    for (int locked = 0; ; locked++) {
+        jl_array_t *speckeyset = jl_atomic_load_acquire(&m->speckeyset);
+        jl_svec_t *specializations = jl_atomic_load_acquire(&m->specializations);
+        size_t i, cl = jl_svec_len(specializations);
+        if (hv) {
+            ssize_t idx = jl_smallintset_lookup(speckeyset, speccache_eq, type, specializations, hv);
+            if (idx != -1) {
+                jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, idx);
+                JL_GC_PROMISE_ROOTED(mi); // clang-sa doesn't understand jl_atomic_load_relaxed
+                if (locked)
+                    JL_UNLOCK(&m->writelock);
                 return mi;
+            }
         }
-    }
-    JL_LOCK(&m->writelock);
-    if (hv) {
-        jl_method_instance_t *mi = (jl_method_instance_t*)jl_typeset_lookup(&m->specializations, speccache_eq, type, hv, jl_nothing);
-        if (!jl_is_method_instance(mi)) {
-            mi = jl_get_specialized(m, type, sparams);
-            JL_GC_PUSH1(&mi);
-            // TODO: fuse lookup and insert steps?
-            jl_typeset_insert(&m->specializations, (jl_value_t*)m, speccache_hash, (jl_value_t*)mi);
-            JL_GC_PROMISE_ROOTED(mi); // clang-sa doesn't understand jl_typeset_insert
+        else {
+            jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations);
+            JL_GC_PUSH1(&specializations); // clang-sa doesn't realize this loop uses specializations
+            for (i = cl; i > 0; i--) {
+                jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i - 1]);
+                JL_GC_PROMISE_ROOTED(mi); // clang-sa doesn't understand jl_atomic_load_relaxed
+                if (mi == NULL)
+                    break;
+                if (jl_types_equal(mi->specTypes, type)) {
+                    if (locked)
+                        JL_UNLOCK(&m->writelock);
+                    JL_GC_POP();
+                    return mi;
+                }
+            }
             JL_GC_POP();
         }
-        JL_UNLOCK(&m->writelock);
-        return mi;
-    }
-    else {
-        jl_svec_t *linearcache = jl_atomic_load_relaxed(&m->linearspecializations);
-        jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(linearcache);
-        size_t i, cl = jl_svec_len(linearcache);
-        for (i = 0; i < cl; i++) {
-            jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
-            JL_GC_PROMISE_ROOTED(mi);
-            if (!mi)
-                break;
-            if (jl_types_equal(mi->specTypes, type)) {
-                JL_UNLOCK(&m->writelock);
-                return mi;
-            }
+        if (!sparams) // can't insert without knowing this
+            return NULL;
+        if (!locked) {
+            JL_LOCK(&m->writelock);
         }
-        if (i == cl) {
-            jl_svec_t *nc = jl_alloc_svec(cl < 8 ? 8 : (cl*3)>>1);
-            memcpy(jl_svec_data(nc), jl_svec_data(linearcache), sizeof(void*) * cl);
-            jl_atomic_store_release(&m->linearspecializations, nc);
-            JL_GC_PROMISE_ROOTED(nc); // clang-sa doesn't understand jl_atomic_store_release
-            jl_gc_wb(m, nc);
-            linearcache = nc;
-            cl = jl_svec_len(nc);
-            data = (jl_method_instance_t**)jl_svec_data(linearcache);
+        else {
+            if (hv) {
+                jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations);
+                for (i = 0; i < cl; i++) {
+                    jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
+                    JL_GC_PROMISE_ROOTED(mi); // clang-sa doesn't understand jl_atomic_load_relaxed
+                    if (mi == NULL)
+                        break;
+                    assert(!jl_types_equal(mi->specTypes, type));
+                }
+            }
+            jl_method_instance_t *mi = jl_get_specialized(m, type, sparams);
+            JL_GC_PUSH1(&mi);
+            if (hv ? (i + 1 >= cl || jl_svecref(specializations, i + 1) != NULL) : (i <= 1 || jl_svecref(specializations, i - 2) != NULL)) {
+                size_t ncl = cl < 8 ? 8 : (cl*3)>>1;
+                jl_svec_t *nc = jl_alloc_svec_uninit(ncl);
+                if (i > 0)
+                    memcpy((char*)jl_svec_data(nc), jl_svec_data(specializations), sizeof(void*) * i);
+                memset((char*)jl_svec_data(nc) + sizeof(void*) * i, 0, sizeof(void*) * (ncl - cl));
+                if (i < cl)
+                    memcpy((char*)jl_svec_data(nc) + sizeof(void*) * (i + ncl - cl),
+                           (char*)jl_svec_data(specializations) + sizeof(void*) * i,
+                           sizeof(void*) * (cl - i));
+                jl_atomic_store_release(&m->specializations, nc);
+                JL_GC_PROMISE_ROOTED(nc); // clang-sa doesn't understand jl_atomic_store_release
+                jl_gc_wb(m, nc);
+                specializations = nc;
+                if (!hv)
+                    i += ncl - cl;
+            }
+            if (!hv)
+                i -= 1;
+            assert(jl_svecref(specializations, i) == NULL);
+            jl_svecset(specializations, i, mi); // jl_atomic_store_release?
+            if (hv) {
+                // TODO: fuse lookup and insert steps?
+                jl_smallintset_insert(&m->speckeyset, (jl_value_t*)m, speccache_hash, i, specializations);
+            }
+            JL_UNLOCK(&m->writelock);
+            JL_GC_POP();
+            return mi;
         }
-        jl_method_instance_t *mi = jl_get_specialized(m, type, sparams);
-        jl_atomic_store_release(&data[i], mi);
-        jl_gc_wb(linearcache, mi);
-        JL_GC_PROMISE_ROOTED(mi); // clang-sa doesn't understand jl_atomic_store_release
-        JL_UNLOCK(&m->writelock);
-        return mi;
     }
 }
 
 JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_value_t *type)
 {
-    uint_t hv = ((jl_datatype_t*)(jl_is_unionall(type) ? jl_unwrap_unionall(type) : type))->hash;
-    if (hv) {
-        return jl_typeset_lookup(&m->specializations, speccache_eq, type, hv, jl_nothing);
-    }
-    else {
-        jl_svec_t *linearcache = jl_atomic_load_relaxed(&m->linearspecializations);
-        jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(linearcache);
-        size_t i, cl = jl_svec_len(linearcache);
-        for (i = 0; i < cl; i++) {
-            jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
-            JL_GC_PROMISE_ROOTED(mi);
-            if (!mi)
-                break;
-            if (jl_types_equal(mi->specTypes, type))
-                return (jl_value_t*)mi;
-        }
+    jl_value_t *mi = (jl_value_t*)jl_specializations_get_linfo(m, type, NULL);
+    if (mi == NULL)
         return jl_nothing;
-    }
+    return mi;
 }
 
 JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *type, size_t world)
@@ -410,16 +415,6 @@ static int get_method_unspec_list(jl_typemap_entry_t *def, void *closure)
                 jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi);
         }
     }
-    specializations = def->func.method->linearspecializations;
-    l = jl_svec_len(specializations);
-    for (i = 0; i < l; i++) {
-        jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
-        if (mi) {
-            assert(jl_is_method_instance(mi));
-            if (jl_rettype_inferred(mi, jl_world_counter, jl_world_counter) == jl_nothing)
-                jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi);
-        }
-    }
     return 1;
 }
 
@@ -1686,13 +1681,6 @@ JL_DLLEXPORT void jl_method_table_disable(jl_methtable_t *mt, jl_method_t *metho
         if (mi)
             invalidate_backedges(mi, methodentry->max_world);
     }
-    specializations = methodentry->func.method->linearspecializations;
-    l = jl_svec_len(specializations);
-    for (i = 0; i < l; i++) {
-        jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
-        if (mi)
-            invalidate_backedges(mi, methodentry->max_world);
-    }
     JL_UNLOCK(&mt->writelock);
 }
 
@@ -1784,14 +1772,6 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method
                     if (invalidate_backedges(mi, max_world))
                         invalidated = 1;
             }
-            specializations = ((jl_method_t*)m)->linearspecializations;
-            l = jl_svec_len(specializations);
-            for (i = 0; i < l; i++) {
-                jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
-                if (mi != NULL && !jl_has_empty_intersection(type, (jl_value_t*)mi->specTypes))
-                    if (invalidate_backedges(mi, max_world))
-                        invalidated = 1;
-            }
         }
     }
     if (invalidated && JL_DEBUG_METHOD_INVALIDATION) {

diff --git a/src/jltypes.c b/src/jltypes.c
@@ -2132,7 +2132,7 @@ void jl_init_types(void) JL_GC_DISABLED
                             "ambig",
                             "resorted",
                             "specializations",
-                            "linearspecializations",
+                            "speckeyset",
                             "slot_syms",
                             "source",
                             "unspecialized",
@@ -2156,7 +2156,7 @@ void jl_init_types(void) JL_GC_DISABLED
                             jl_any_type, // Union{Vector, Nothing}
                             jl_any_type, // Union{Vector, Nothing}
                             jl_simplevector_type,
-                            jl_simplevector_type,
+                            jl_array_type,
                             jl_string_type,
                             jl_any_type,
                             jl_any_type, // jl_method_instance_type

diff --git a/src/julia.h b/src/julia.h
@@ -290,8 +290,8 @@ typedef struct _jl_method_t {
     jl_value_t *resorted;
 
     // table of all jl_method_instance_t specializations we have
-    jl_svec_t *specializations; // any type with a hash
-    jl_svec_t *linearspecializations; // all other types
+    jl_svec_t *specializations; // allocated as [hashable, ..., NULL, linear, ....]
+    jl_array_t *speckeyset; // index lookup by hash into specializations
 
     jl_value_t *slot_syms; // compacted list of slot names (String)
     jl_value_t *source;  // original code template (jl_code_info_t, but may be compressed), null for builtins

diff --git a/src/julia_internal.h b/src/julia_internal.h
@@ -1021,13 +1021,12 @@ extern jl_mutex_t safepoint_lock;
 void jl_mach_gc_end(void);
 #endif
 
-// -- typeset.c -- //
+// -- smallintset.c -- //
 
-typedef uint_t (*typeset_hash)(jl_value_t *val);
-typedef int (*typeset_eq)(jl_value_t *val, const void *key, uint_t hv);
-jl_value_t **jl_typeset_lookup_bp(jl_svec_t **cache JL_PROPAGATES_ROOT, typeset_eq eq, const void *key, uint_t hv, jl_value_t **notfound);
-jl_value_t *jl_typeset_lookup(jl_svec_t **cache JL_PROPAGATES_ROOT, typeset_eq eq, const void *key, uint_t hv, jl_value_t *notfound);
-void jl_typeset_insert(jl_svec_t **cache, jl_value_t *parent, typeset_hash hash, jl_value_t *val);
+typedef uint_t (*smallintset_hash)(size_t val, jl_svec_t *data);
+typedef int (*smallintset_eq)(size_t val, const void *key, jl_svec_t *data, uint_t hv);
+ssize_t jl_smallintset_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv);
+void jl_smallintset_insert(jl_array_t **pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data);
 
 // -- typemap.c -- //
 

diff --git a/src/method.c b/src/method.c
@@ -592,7 +592,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module)
     jl_method_t *m =
         (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), jl_method_type);
     m->specializations = jl_emptysvec;
-    m->linearspecializations = jl_emptysvec;
+    m->speckeyset = (jl_array_t*)jl_an_empty_vec_any;
     m->sig = NULL;
     m->slot_syms = NULL;
     m->ambig = jl_nothing;

diff --git a/src/precompile.c b/src/precompile.c
@@ -358,13 +358,6 @@ static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *c
             if (mi != NULL)
                 precompile_enq_specialization_(mi, closure);
         }
-        specializations = def->func.method->linearspecializations;
-        l = jl_svec_len(specializations);
-        for (i = 0; i < l; i++) {
-            jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
-            if (mi != NULL)
-                precompile_enq_specialization_(mi, closure);
-        }
     }
     return 1;
 }