diff --git a/base/reflection.jl b/base/reflection.jl index eaa802f737fcd..9635783602c84 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -910,6 +910,7 @@ function visit(f, mt::Core.MethodTable) nothing end function visit(f, mc::Core.TypeMapLevel) + mc.bottom !== nothing && visit(f, mc.bottom) if mc.targ !== nothing e = mc.targ::Vector{Any} for i in 2:2:length(e) @@ -922,6 +923,20 @@ function visit(f, mc::Core.TypeMapLevel) isassigned(e, i) && visit(f, e[i]) end end + if mc.tname !== nothing + for e in mc.tname::SimpleVector + e = e::Vector{Any} + for i in 2:2:length(e) + isassigned(e, i) && visit(f, e[i]) + end + end + end + if mc.name1 !== nothing + e = mc.name1::Vector{Any} + for i in 2:2:length(e) + isassigned(e, i) && visit(f, e[i]) + end + end mc.list !== nothing && visit(f, mc.list) mc.any !== nothing && visit(f, mc.any) nothing diff --git a/src/datatype.c b/src/datatype.c index d22551895d99a..323d9b364f7b7 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -55,7 +55,7 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo mt->kwsorter = NULL; mt->backedges = NULL; JL_MUTEX_INIT(&mt->writelock); - mt->offs = 1; + mt->offs = 0; mt->frozen = 0; return mt; } @@ -569,8 +569,8 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( // as an optimization tn->mt = jl_new_method_table(name, module); jl_gc_wb(tn, tn->mt); - if (jl_svec_len(parameters) > 0) - tn->mt->offs = 0; + if (jl_svec_len(parameters) == 0 && !abstract) + tn->mt->offs = 1; } else { // Everything else, gets to use the unified table diff --git a/src/dump.c b/src/dump.c index 09472aa60f6f5..597ef96798756 100644 --- a/src/dump.c +++ b/src/dump.c @@ -783,22 +783,6 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li } return; } - if (t == jl_typemap_level_type) { - // perform some compression on the typemap levels - // (which will need to be rehashed during deserialization anyhow) - jl_typemap_level_t *node = (jl_typemap_level_t*)v; - assert( // make sure this type has the expected ordering and layout - offsetof(jl_typemap_level_t, arg1) == 0 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, targ) == 1 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, linear) == 2 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, any) == 3 * sizeof(jl_value_t*) && - sizeof(jl_typemap_level_t) == 4 * sizeof(jl_value_t*)); - jl_serialize_value(s, node->arg1); - jl_serialize_value(s, node->targ); - jl_serialize_value(s, node->linear); - jl_serialize_value(s, node->any); - return; - } char *data = (char*)jl_data_ptr(v); size_t i, j, np = t->layout->npointers; diff --git a/src/jltypes.c b/src/jltypes.c index 64a6fb804ffa9..474abf1004db8 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1871,7 +1871,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type->super = jl_any_type; jl_nonfunction_mt = jl_any_type->name->mt; jl_any_type->name->mt = NULL; - jl_nonfunction_mt->offs = 0; jl_type_type = (jl_unionall_t*)jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), core, jl_any_type, jl_emptysvec); jl_type_typename = ((jl_datatype_t*)jl_type_type)->name; @@ -2089,17 +2088,23 @@ void jl_init_types(void) JL_GC_DISABLED jl_typemap_level_type = jl_new_datatype(jl_symbol("TypeMapLevel"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(4, + jl_perm_symsvec(7, + "bottom", "arg1", "targ", + "name1", + "tname", "list", "any"), - jl_svec(4, + jl_svec(7, + jl_any_type, + jl_any_type, + jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type), - 0, 1, 4); + 0, 1, 7); jl_typemap_entry_type = jl_new_datatype(jl_symbol("TypeMapEntry"), core, jl_any_type, jl_emptysvec, diff --git a/src/julia.h b/src/julia.h index 86f7d7a0380dd..f699cdf9a7821 100644 --- a/src/julia.h +++ b/src/julia.h @@ -523,14 +523,24 @@ typedef struct _jl_typemap_entry_t { int8_t va; // isVararg(sig) } jl_typemap_entry_t; -// one level in a TypeMap tree -// indexed by key if it is a sublevel in an array +// one level in a TypeMap tree (each level splits on a type at a given offset) typedef struct _jl_typemap_level_t { JL_DATA_TYPE - jl_array_t *arg1; - jl_array_t *targ; - jl_typemap_entry_t *linear; // jl_typemap_t * (but no more levels) - jl_typemap_t *any; // type at offs is Any + // these vectors contains vectors of more levels in their intended visit order + // with an index that gives the functionality of a sorted dict. + // The first entry may be bottom (for entries which have no type at offs) + // next split may be on Type{T} as LeafTypes then TypeNames parents up to Any + // next split may be on LeafType + // next split may be on TypeName + jl_typemap_entry_t *bottom; // jl_typemap_t * (but no more levels) for entries which have no type at offs + jl_array_t *arg1; // contains LeafType + jl_array_t *targ; // contains Type{LeafType} + jl_array_t *name1; // contains non-abstract TypeName + jl_svec_t *tname; // contains a list of Type{TypeName} dicts, for parents up to Any + // next a linear list of unsortable things (no more levels) + jl_typemap_entry_t *linear; + // finally, start a new level if the type at offs is Any + jl_typemap_t *any; } jl_typemap_level_t; // contains the TypeMap for one Type diff --git a/src/julia_internal.h b/src/julia_internal.h index 313a9538bc302..493faa794a6e8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1038,7 +1038,7 @@ void jl_mach_gc_end(void); typedef uint_t (*smallintset_hash)(size_t val, jl_svec_t *data); typedef int (*smallintset_eq)(size_t val, const void *key, jl_svec_t *data, uint_t hv); -ssize_t jl_smallintset_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv); +ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv); void jl_smallintset_insert(jl_array_t **pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data); // -- typemap.c -- // diff --git a/src/rtutils.c b/src/rtutils.c index 7008691b18908..4e6a8342c5241 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -664,6 +664,10 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_static_show_x(out, li->uninferred, depth); } } + else if (vt == jl_typename_type) { + n += jl_static_show_x(out, jl_unwrap_unionall(((jl_typename_t*)v)->wrapper), depth); + n += jl_printf(out, ".name"); + } else if (vt == jl_simplevector_type) { n += jl_show_svec(out, (jl_svec_t*)v, "svec", "(", ")"); } diff --git a/src/typemap.c b/src/typemap.c index 1f8192ef0a019..12bb0273c897e 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -19,9 +19,80 @@ extern "C" { // compute whether the specificity of this type is equivalent to Any in the sort order static int jl_is_any(jl_value_t *t1) { - return (t1 == (jl_value_t*)jl_any_type || - (jl_is_typevar(t1) && - ((jl_tvar_t*)t1)->ub == (jl_value_t*)jl_any_type)); + while (jl_is_typevar(t1)) + t1 = ((jl_tvar_t*)t1)->ub; + return t1 == (jl_value_t*)jl_any_type; +} + +static jl_value_t *jl_type_extract_name(jl_value_t *t1, int abstract) +{ + if (jl_is_unionall(t1)) + t1 = jl_unwrap_unionall(t1); + if (jl_is_vararg_type(t1)) { + return jl_type_extract_name(jl_tparam0(t1), abstract); + } + else if (jl_is_typevar(t1)) { + return jl_type_extract_name(((jl_tvar_t*)t1)->ub, abstract); + } + else if (jl_is_datatype(t1)) { + jl_datatype_t *dt = (jl_datatype_t*)t1; + if ((abstract || !dt->abstract) && !jl_is_kind(t1)) + return (jl_value_t*)dt->name; + return NULL; + } + else if (jl_is_uniontype(t1)) { + jl_uniontype_t *u1 = (jl_uniontype_t*)t1; + jl_value_t *tn1 = jl_type_extract_name(u1->a, abstract); + jl_value_t *tn2 = jl_type_extract_name(u1->b, abstract); + if (tn1 == tn2) + return tn1; + // TODO: if abstract is true, instead find the nearest common ancester + return NULL; + } + return NULL; +} + +// return true if the name extracted above is an over-approximation +// (such that intersection also needs to consider the subtypes) +static int jl_type_extract_name_imprecise(jl_value_t *t1, int abstract) +{ + if (jl_is_unionall(t1)) + t1 = jl_unwrap_unionall(t1); + if (jl_is_vararg_type(t1)) { + return jl_type_extract_name_imprecise(jl_tparam0(t1), abstract); + } + else if (jl_is_typevar(t1)) { + return jl_type_extract_name_imprecise(((jl_tvar_t*)t1)->ub, 0); + } + else if (jl_is_datatype(t1)) { + jl_datatype_t *dt = (jl_datatype_t*)t1; + if ((abstract || !dt->abstract) && !jl_is_kind(t1)) + return 0; + return 1; + } + else if (jl_is_uniontype(t1)) { + jl_uniontype_t *u1 = (jl_uniontype_t*)t1; + if (jl_type_extract_name_imprecise(u1->a, abstract)) + return 1; + if (jl_type_extract_name_imprecise(u1->b, abstract)) + return 1; + jl_value_t *tn1 = jl_type_extract_name(u1->a, abstract); + jl_value_t *tn2 = jl_type_extract_name(u1->b, abstract); + if (tn1 == tn2) + return 0; + return 1; + } + return 0; +} + +static unsigned jl_supertype_height(jl_datatype_t *dt) +{ + unsigned height = 1; + while (dt != jl_any_type) { + height++; + dt = dt->super; + } + return height; } // ----- Type Signature Subtype Testing ----- // @@ -183,11 +254,14 @@ static inline int sig_match_simple(jl_value_t *arg1, jl_value_t **args, size_t n // predicate to fast-test if this type is a leaf type that can exist in the cache // and does not need a more expensive linear scan to find all intersections -// be careful not to put non-leaf types or DataType/UnionAll in the cache here, -// since they should have a lower priority and need to go into the sorted list -int is_cache_leaf(jl_value_t *ty) +// be careful not to put non-leaf types or DataType/UnionAll/Union in the +// argument cache, since they should have a lower priority and so will go in some +// later list +int is_cache_leaf(jl_value_t *ty, int tparam) { - return (jl_is_concrete_type(ty) && !jl_is_kind(ty)); + if (ty == jl_bottom_type) + return 1; + return (jl_is_concrete_type(ty) && (tparam || !jl_is_kind(ty))); } static jl_typemap_t **mtcache_hash_lookup_bp(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT @@ -219,7 +293,7 @@ static void mtcache_hash_insert(jl_array_t **cache, jl_value_t *parent, jl_value static jl_typemap_t *mtcache_hash_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT { if (cache == (jl_array_t*)jl_an_empty_vec_any) - return NULL; + return (jl_typemap_t*)jl_nothing; jl_typemap_t *ml = (jl_typemap_t*)jl_eqtable_get(cache, ty, jl_nothing); JL_GC_PROMISE_ROOTED(ml); // clang-sa doesn't trust our JL_PROPAGATES_ROOT claim return ml; @@ -240,6 +314,18 @@ static int jl_typemap_array_visitor(jl_array_t *a, jl_typemap_visitor_fptr fptr, return 1; } +static int jl_typemap_svec_visitor(jl_svec_t *a, jl_typemap_visitor_fptr fptr, void *closure) +{ + size_t i, l = jl_svec_len(a); + for (i = 0; i < l; i++) { + jl_array_t *d = (jl_array_t*)jl_svecref(a, i); + if (!jl_typemap_array_visitor(d, fptr, closure)) + return 0; + } + return 1; +} + + // calls fptr on each jl_typemap_entry_t in cache in sort order, until fptr return false static int jl_typemap_node_visitor(jl_typemap_entry_t *ml, jl_typemap_visitor_fptr fptr, void *closure) { @@ -255,12 +341,21 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * { if (jl_typeof(cache) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *node = (jl_typemap_level_t*)cache; + if (node->bottom != (void*)jl_nothing) + if (!jl_typemap_node_visitor(node->bottom, fptr, closure)) + return 0; if (node->targ != (jl_array_t*)jl_an_empty_vec_any) if (!jl_typemap_array_visitor(node->targ, fptr, closure)) return 0; if (node->arg1 != (jl_array_t*)jl_an_empty_vec_any) if (!jl_typemap_array_visitor(node->arg1, fptr, closure)) return 0; + if (node->tname != jl_emptysvec) + if (!jl_typemap_svec_visitor(node->tname, fptr, closure)) + return 0; + if (node->name1 != (jl_array_t*)jl_an_empty_vec_any) + if (!jl_typemap_array_visitor(node->name1, fptr, closure)) + return 0; if (!jl_typemap_node_visitor(node->linear, fptr, closure)) return 0; return jl_typemap_visitor(node->any, fptr, closure); @@ -280,16 +375,24 @@ static int jl_typemap_intersection_array_visitor(jl_array_t *a, jl_value_t *ty, JL_GC_PROMISE_ROOTED(t); if (t == jl_nothing || t == NULL) continue; - // `t` is a leaftype, so intersection test becomes subtype - if (ty == (jl_value_t*)jl_any_type || // easy case: Any always matches - (tparam - ? (jl_typeof(t) == ty || jl_isa(t, ty)) // (Type{t} <: ty), where is_leaf_type(t) => isa(t, ty) - : (t == ty || jl_subtype(t, ty)))) { + if (tparam & 2) { jl_typemap_t *ml = data[i + 1]; JL_GC_PROMISE_ROOTED(ml); if (!jl_typemap_intersection_visitor(ml, offs + 1, closure)) return 0; } + else { + // `t` is a leaftype, so intersection test becomes subtype + if (ty == (jl_value_t*)jl_any_type || // easy case: Any always matches + (tparam & 1 + ? (jl_typeof(t) == ty || jl_isa(t, ty)) // (Type{t} <: ty), where is_leaf_type(t) => isa(t, ty) + : (t == ty || jl_subtype(t, ty)))) { + jl_typemap_t *ml = data[i + 1]; + JL_GC_PROMISE_ROOTED(ml); + if (!jl_typemap_intersection_visitor(ml, offs + 1, closure)) + return 0; + } + } } return 1; } @@ -354,17 +457,28 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, //} if (jl_typeof(map) == (jl_value_t *)jl_typemap_level_type) { jl_typemap_level_t *cache = (jl_typemap_level_t*)map; - jl_value_t *ty = NULL; + jl_value_t *ty; size_t l = jl_nparams(ttypes); if (closure->va && l <= offs + 1) { ty = closure->va; + if (cache->bottom != (jl_typemap_entry_t*)jl_nothing) { + if (!jl_typemap_intersection_node_visitor(cache->bottom, closure)) return 0; + } } else if (l > offs) { ty = jl_tparam(ttypes, offs); } + else { + if (cache->bottom != (jl_typemap_entry_t*)jl_nothing) { + if (!jl_typemap_intersection_node_visitor(cache->bottom, closure)) return 0; + } + ty = NULL; + } if (ty) { while (jl_is_typevar(ty)) ty = ((jl_tvar_t*)ty)->ub; + jl_value_t *typetype = jl_unwrap_unionall(ty); + typetype = jl_is_type_type(typetype) ? jl_tparam0(typetype) : NULL; // approxify the tparam until we have a valid type if (jl_has_free_typevars(ty)) { ty = jl_unwrap_unionall(ty); @@ -374,9 +488,8 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, ty = (jl_value_t*)jl_any_type; } if (cache->targ != (jl_array_t*)jl_an_empty_vec_any) { - jl_value_t *typetype = jl_is_type_type(ty) ? jl_tparam0(ty) : NULL; if (typetype && !jl_has_free_typevars(typetype)) { - if (is_cache_leaf(typetype)) { + if (is_cache_leaf(typetype, 1)) { // direct lookup of leaf types jl_typemap_t *ml = mtcache_hash_lookup(cache->targ, typetype); if (ml != jl_nothing) { @@ -392,7 +505,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } if (cache->arg1 != (jl_array_t*)jl_an_empty_vec_any) { - if (is_cache_leaf(ty)) { + if (is_cache_leaf(ty, 0)) { // direct lookup of leaf types jl_typemap_t *ml = mtcache_hash_lookup(cache->arg1, ty); if (ml != jl_nothing) { @@ -404,6 +517,64 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0; } } + if (cache->tname != jl_emptysvec) { + jl_value_t *name = typetype ? jl_type_extract_name(typetype, 1) : NULL; + jl_svec_t *tname = cache->tname; + if (name && !jl_is_typevar(typetype)) { + // semi-direct lookup of types + // TODO: the possibility of encountering `Type{Union{}}` in this intersection may + // be forcing us to do some extra work here whenever we see a typevar, even though + // the likelyhood of that value actually occurring is/ frequently likely to be + // zero (or result in an ambiguous match) + jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); + unsigned height = jl_supertype_height(super); + if (height < jl_svec_len(tname) && jl_type_extract_name_imprecise(typetype, 1)) { + // first consider all of the possible subtypes + size_t j; + for (j = 0; j < height; j++) { + if (!jl_typemap_intersection_array_visitor((jl_array_t*)jl_svecref(tname, j), ty, 3, offs, closure)) return 0; + } + } + // then consider the type and its direct super types + while (1) { + if (height <= jl_svec_len(tname)) { + jl_typemap_t *ml = mtcache_hash_lookup((jl_array_t*)jl_svecref(tname, jl_svec_len(tname) - height), (jl_value_t*)super->name); + if (ml != jl_nothing) { + if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0; + } + } + height--; + if (super == jl_any_type) + break; + super = super->super; + } + assert(height == 0); + } + else { + // else an array scan is required to check subtypes + // first, fast-path: optimized pre-intersection test to see if `ty` could intersect with any Type + if (!jl_has_empty_intersection((jl_value_t*)jl_type_type, ty)) { + size_t j; + for (j = 0; j < jl_svec_len(tname); j++) { + if (!jl_typemap_intersection_array_visitor((jl_array_t*)jl_svecref(tname, j), ty, 3, offs, closure)) return 0; + } + } + } + } + if (cache->name1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_value_t *name = jl_type_extract_name(ty, 0); + if (name) { + // direct lookup of concrete types + jl_typemap_t *ml = mtcache_hash_lookup(cache->name1, name); + if (ml != jl_nothing) { + if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0; + } + } + else { + // else an array scan is required to check subtypes + if (!jl_typemap_intersection_array_visitor(cache->name1, ty, 2, offs, closure)) return 0; + } + } } if (!jl_typemap_intersection_node_visitor(cache->linear, closure)) return 0; @@ -551,7 +722,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (jl_typeof(ml_or_cache) == (jl_value_t *)jl_typemap_level_type) { jl_typemap_level_t *cache = (jl_typemap_level_t*)ml_or_cache; // called object is the primary key for constructors, otherwise first argument - jl_value_t *ty = NULL; + jl_value_t *ty; jl_value_t *ttypes = jl_unwrap_unionall((jl_value_t*)search->types); JL_GC_PROMISE_ROOTED(ttypes); assert(jl_is_datatype(ttypes)); @@ -563,25 +734,41 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (jl_is_vararg_type(ty)) { ty = jl_unwrap_vararg(ty); isva = 1; + jl_typemap_entry_t *li = subtype ? + jl_typemap_entry_assoc_by_type(cache->bottom, search) : + jl_typemap_entry_lookup_by_type(cache->bottom, search); + if (li) return li; } else if (l <= offs) { + jl_typemap_entry_t *li = subtype ? + jl_typemap_entry_assoc_by_type(cache->bottom, search) : + jl_typemap_entry_lookup_by_type(cache->bottom, search); + if (li) return li; ty = NULL; } } else if (l > offs) { ty = jl_tparam(ttypes, offs); } - // If there is a type at offs, look in the optimized caches - if (!subtype) { - if (ty && jl_is_any(ty)) + else { + jl_typemap_entry_t *li = subtype ? + jl_typemap_entry_assoc_by_type(cache->bottom, search) : + jl_typemap_entry_lookup_by_type(cache->bottom, search); + if (li) return li; + ty = NULL; + } + // If there is a type at offs, look in the optimized leaf type caches + if (ty && !subtype) { + if (jl_is_any(ty)) return jl_typemap_assoc_by_type(cache->any, search, offs + 1, subtype); if (isva) // in lookup mode, want to match Vararg exactly, not as a subtype ty = NULL; } if (ty) { + // now look at the optimized leaftype caches if (jl_is_type_type(ty)) { jl_value_t *a0 = jl_tparam0(ty); - if (is_cache_leaf(a0)) { + if (is_cache_leaf(a0, 1)) { if (cache->targ != (jl_array_t*)jl_an_empty_vec_any) { jl_typemap_t *ml = mtcache_hash_lookup(cache->targ, a0); if (ml != jl_nothing) { @@ -592,7 +779,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (!subtype) return NULL; } } - if (is_cache_leaf(ty)) { + if (is_cache_leaf(ty, 1)) { if (cache->arg1 != (jl_array_t*)jl_an_empty_vec_any) { jl_typemap_t *ml = mtcache_hash_lookup(cache->arg1, ty); if (ml != jl_nothing) { @@ -603,6 +790,75 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (!subtype) return NULL; } } + if (ty || subtype) { + // now look at the optimized TypeName caches + if (cache->tname != jl_emptysvec) { + jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty), 1) : NULL; + if (a0) { + jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); + unsigned height = jl_supertype_height(super); + while (1) { + if (height <= jl_svec_len(cache->tname)) { + jl_typemap_t *ml = mtcache_hash_lookup((jl_array_t*)jl_svecref(cache->tname, jl_svec_len(cache->tname) - height), (jl_value_t*)super->name); + if (ml != (void*)jl_nothing) { + jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, search, offs + 1, subtype); + if (li) return li; + } + } + height--; + if (super == jl_any_type) + break; + super = super->super; + } + assert(height == 0); + } + else { + if (!ty || !jl_has_empty_intersection((jl_value_t*)jl_type_type, ty)) { + // couldn't figure out unique `a0` initial point, so scan all for matches + jl_svec_t *tname = cache->tname; + size_t j; + for (j = 0; j < jl_svec_len(tname); j++) { + jl_array_t *a = (jl_array_t*)jl_svecref(tname, j); + size_t i, l = jl_array_len(a); + jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(a); + for (i = 1; i < l; i += 2) { + jl_typemap_t *ml = data[i]; + if (ml == NULL || ml == jl_nothing) + continue; + jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, search, offs + 1, subtype); + if (li) return li; + } + } + } + } + } + if (cache->name1 != (jl_array_t*)jl_an_empty_vec_any) { + if (ty) { + jl_value_t *a0 = jl_type_extract_name(ty, 0); + if (a0) { + jl_typemap_t *ml = mtcache_hash_lookup(cache->name1, a0); + if (ml != (void*)jl_nothing) { + jl_typemap_entry_t *li = + jl_typemap_assoc_by_type(ml, search, offs + 1, subtype); + if (li) return li; + } + } + } + else { + // doing subtype, but couldn't figure out unique `ty`, so scan all for supertypes + jl_array_t *a = cache->name1; + size_t i, l = jl_array_len(a); + jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(a); + for (i = 1; i < l; i += 2) { + jl_typemap_t *ml = data[i]; + if (ml == NULL || ml == jl_nothing) + continue; + jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, search, offs + 1, subtype); + if (li) return li; + } + } + } + } // Always check the list (since offs doesn't always start at 0) if (subtype) { jl_typemap_entry_t *li = jl_typemap_entry_assoc_by_type(cache->linear, search); @@ -701,20 +957,53 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world) { - if (n > offs) { + if (n <= offs) { + if (cache->bottom != (jl_typemap_entry_t*)jl_nothing) { + jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->bottom, arg1, args, n, world); + if (ml) return ml; + } + } + else { jl_value_t *a1 = (offs == 0 ? arg1 : args[offs - 1]); jl_value_t *ty = jl_typeof(a1); assert(jl_is_datatype(ty)); - if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(a1)) { + if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(a1, 1)) { jl_typemap_t *ml_or_cache = mtcache_hash_lookup(cache->targ, a1); jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } - if (cache->arg1 != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(ty)) { + if (cache->arg1 != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(ty, 0)) { jl_typemap_t *ml_or_cache = mtcache_hash_lookup(cache->arg1, ty); jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } + if (jl_is_kind(ty) && cache->tname != jl_emptysvec) { + jl_value_t *name = jl_type_extract_name(a1, 1); + if (name) { + if (ty != (jl_value_t*)jl_datatype_type) + a1 = jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); + unsigned height = jl_supertype_height((jl_datatype_t*)a1); + while (1) { + if (height <= jl_svec_len(cache->tname)) { + jl_typemap_t *ml_or_cache = mtcache_hash_lookup( + (jl_array_t*)jl_svecref(cache->tname, jl_svec_len(cache->tname) - height), + (jl_value_t*)((jl_datatype_t*)a1)->name); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); + if (ml) return ml; + } + height--; + if (a1 == (jl_value_t*)jl_any_type) + break; + a1 = (jl_value_t*)((jl_datatype_t*)a1)->super; + } + assert(height == 0); + } + } + if (cache->name1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_typemap_t *ml_or_cache = mtcache_hash_lookup(cache->name1, (jl_value_t*)((jl_datatype_t*)ty)->name); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); + if (ml) return ml; + } } if (cache->linear != (jl_typemap_entry_t*)jl_nothing) { jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, arg1, args, n, world); @@ -749,10 +1038,13 @@ static jl_typemap_level_t *jl_new_typemap_level(void) jl_typemap_level_t *cache = (jl_typemap_level_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_level_t), jl_typemap_level_type); + cache->bottom = (jl_typemap_entry_t*)jl_nothing; + cache->arg1 = (jl_array_t*)jl_an_empty_vec_any; + cache->targ = (jl_array_t*)jl_an_empty_vec_any; + cache->name1 = (jl_array_t*)jl_an_empty_vec_any; + cache->tname = jl_emptysvec; cache->linear = (jl_typemap_entry_t*)jl_nothing; cache->any = jl_nothing; - cache->targ = (jl_array_t*)jl_an_empty_vec_any; - cache->arg1 = (jl_array_t*)jl_an_empty_vec_any; return cache; } @@ -814,19 +1106,16 @@ static void jl_typemap_insert_generic( parent, newrec, tparams); } -static int jl_typemap_array_insert_( +static void jl_typemap_array_insert_( jl_typemap_t *map, jl_array_t **cache, jl_value_t *key, jl_typemap_entry_t *newrec, - jl_value_t *parent, int8_t tparam, int8_t offs, + jl_value_t *parent, int8_t offs, const struct jl_typemap_info *tparams) { - if (!is_cache_leaf(key)) - return 0; jl_typemap_t **pml = mtcache_hash_lookup_bp(*cache, key); if (pml != NULL) jl_typemap_insert_generic(map, pml, (jl_value_t*)*cache, newrec, offs+1, tparams); else mtcache_hash_insert(cache, parent, key, (jl_typemap_t*)newrec); - return 1; } static void jl_typemap_level_insert_( @@ -836,7 +1125,7 @@ static void jl_typemap_level_insert_( jl_value_t *ttypes = jl_unwrap_unionall((jl_value_t*)newrec->sig); size_t l = jl_nparams(ttypes); // compute the type at offset `offs` into `sig`, which may be a Vararg - jl_value_t *t1 = NULL; + jl_value_t *t1; int isva = 0; if (l <= offs + 1) { t1 = jl_tparam(ttypes, l - 1); @@ -851,6 +1140,13 @@ static void jl_typemap_level_insert_( else if (l > offs) { t1 = jl_tparam(ttypes, offs); } + else { + t1 = NULL; + } + if (0 && t1 == NULL) { + jl_typemap_list_insert_(map, &cache->bottom, (jl_value_t*)cache, newrec, tparams); + return; + } // If the type at `offs` is Any, put it in the Any list if (t1 && jl_is_any(t1)) { jl_typemap_insert_generic(map, &cache->any, (jl_value_t*)cache, newrec, offs+1, tparams); @@ -858,16 +1154,46 @@ static void jl_typemap_level_insert_( } // Don't put Varargs in the optimized caches (too hard to handle in lookup and bp) if (t1 && !isva) { - // if t1 != jl_typetype_type and the argument is Type{...}, this - // method has specializations for singleton kinds and we use - // the table indexed for that purpose. + // try to put in leaf type caches if (t1 != (jl_value_t*)jl_typetype_type && jl_is_type_type(t1)) { + // if t1 != jl_typetype_type and the argument is Type{...}, this + // method has specializations for singleton kinds and we use + // the table indexed for that purpose. jl_value_t *a0 = jl_tparam0(t1); - if (jl_typemap_array_insert_(map, &cache->targ, a0, newrec, (jl_value_t*)cache, 1, offs, tparams)) + if (is_cache_leaf(a0, 1)) { + jl_typemap_array_insert_(map, &cache->targ, a0, newrec, (jl_value_t*)cache, offs, tparams); return; + } } - if (jl_typemap_array_insert_(map, &cache->arg1, t1, newrec, (jl_value_t*)cache, 0, offs, tparams)) + if (is_cache_leaf(t1, 0)) { + jl_typemap_array_insert_(map, &cache->arg1, t1, newrec, (jl_value_t*)cache, offs, tparams); return; + } + + // try to put in TypeName caches + jl_value_t *a0; + t1 = jl_unwrap_unionall(t1); + if (jl_is_type_type(t1)) { + a0 = jl_type_extract_name(jl_tparam0(t1), 1); + jl_datatype_t *super = a0 ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper) : jl_any_type; + unsigned height = jl_supertype_height(super); + if (height > jl_svec_len(cache->tname)) { + jl_svec_t *newcache = jl_svec_fill(height, jl_an_empty_vec_any); + memcpy(jl_svec_data(newcache) + height - jl_svec_len(cache->tname), + jl_svec_data(cache->tname), + jl_svec_len(cache->tname) * sizeof(void*)); + cache->tname = newcache; + jl_gc_wb(cache, newcache); + } + jl_typemap_array_insert_(map, (jl_array_t**)&jl_svec_data(cache->tname)[jl_svec_len(cache->tname) - height], + (jl_value_t*)super->name, newrec, (jl_value_t*)cache->tname, offs, tparams); + return; + } + a0 = jl_type_extract_name(t1, 0); + if (a0) { + jl_typemap_array_insert_(map, &cache->name1, a0, newrec, (jl_value_t*)cache, offs, tparams); + return; + } } jl_typemap_list_insert_(map, &cache->linear, (jl_value_t*)cache, newrec, tparams); }