cfunction: redesign cache support

Provides a simpler fast path (no allocations and rebuilding type vectors
required to search the cache).

Author: vtjnash

src/ccall.cpp

@@ -1774,13 +1774,13 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
     else if (is_libjulia_func(jl_function_ptr)) {
         assert(!isVa && !llvmcall && nargt == 3);
         assert(lrt == T_size);
-        jl_value_t *f = argv[0].constant;
+        jl_value_t *ft = argv[0].typ;
         jl_value_t *frt = argv[1].constant;
         if (!frt) {
             if (jl_is_type_type(argv[1].typ) && !jl_has_free_typevars(argv[1].typ))
                 frt = jl_tparam0(argv[1].typ);
         }
-        if (f && frt) {
+        if (ft != jl_bottom_type && frt) {
             jl_value_t *fargt = argv[2].constant;;
             JL_GC_PUSH1(&fargt);
             if (!fargt && jl_is_type_type(argv[2].typ)) {
@@ -1794,7 +1794,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
             if (fargt && jl_is_tuple_type(fargt)) {
                 Value *llvmf = NULL;
                 JL_TRY {
-                    llvmf = jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt);
+                    llvmf = jl_cfunction_cache(ft, frt, (jl_tupletype_t*)fargt);
                 }
                 JL_CATCH {
                     llvmf = NULL;

src/codegen.cpp

@@ -1414,7 +1414,20 @@ jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t *li, const char *F, size
     return fptr;
 }
 
-static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt);
+static Function *jl_cfunction_cache(jl_value_t *ft, jl_value_t *declrt, jl_tupletype_t *argt);
+
+// Get a pointer to the cache for the C-callable entry point for the given argument types.
+// here argt does not include the leading function type argument
+static Function *jl_cfunction_object(jl_value_t *f, jl_value_t *declrt, jl_tupletype_t *argt)
+{
+    jl_value_t *ft;
+    if (jl_is_type(f))
+        ft = (jl_value_t*)jl_wrap_Type(f);
+    else
+        ft = jl_typeof(f);
+    return jl_cfunction_cache(ft, declrt, argt);
+}
+
 // get the address of a C-callable entry point for a function
 extern "C" JL_DLLEXPORT
 void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt)
@@ -1443,7 +1456,6 @@ void *jl_function_ptr_by_llvm_name(char *name) {
 extern "C" JL_DLLEXPORT
 void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name)
 {
-    assert(jl_is_tuple_type(argt));
     JL_LOCK(&codegen_lock);
     Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt);
     // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation)
@@ -4168,9 +4180,9 @@ static void emit_cfunc_invalidate(
     }
 }
 
-static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
-                                  jl_value_t *jlrettype, jl_tupletype_t *argt,
-                                  jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt)
+static Function*
+gen_cfun_wrapper(const function_sig_t &sig, jl_value_t *ff,
+    jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt)
 {
     // Generate a c-callable wrapper
     size_t nargs = sig.nargs;
@@ -4221,8 +4233,19 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
 #endif
     Function *cw_proto = function_proto(cw);
     // Save the Function object reference
-    sf->func.value = jl_box_voidpointer((void*)cw_proto);
-    jl_gc_wb(sf, sf->func.value);
+    {
+        jl_value_t *oldsf = sf->func.value;
+        size_t i, oldlen = jl_svec_len(oldsf);
+        jl_value_t *newsf = (jl_value_t*)jl_alloc_svec(oldlen + 2);
+        JL_GC_PUSH1(&newsf);
+        jl_svecset(newsf, 0, sig.rt);
+        jl_svecset(newsf, 1, jl_box_voidpointer((void*)cw_proto));
+        for (i = 0; i < oldlen; i++)
+            jl_svecset(newsf, i + 2, jl_svecref(oldsf, i));
+        sf->func.value = newsf;
+        jl_gc_wb(sf, sf->func.value);
+        JL_GC_POP();
+    }
 
     jl_codectx_t ctx(jl_LLVMContext);
     ctx.f = cw;
@@ -4266,10 +4289,11 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
     Function::arg_iterator AI = cw->arg_begin();
     Value *sretPtr = sig.sret ? &*AI++ : NULL;
     jl_cgval_t *inputargs = (jl_cgval_t*)alloca(sizeof(jl_cgval_t) * (nargs + 1));
-    inputargs[0] = mark_julia_const(ff); // we need to pass the function object even if (even though) it is a ghost
+    // we need to pass the function object even if (even though) it is a singleton
+    inputargs[0] = mark_julia_const(ff);
     for (size_t i = 0; i < nargs; ++i, ++AI) {
         Value *val = &*AI;
-        jl_value_t *jargty = jl_nth_slot_type((jl_value_t*)argt, i);
+        jl_value_t *jargty = jl_svecref(sig.at, i);
         // figure out how to unpack this type
         jl_cgval_t &inputarg = inputargs[i + 1];
         if (jl_is_abstract_ref_type(jargty)) {
@@ -4521,90 +4545,112 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
 }
 
 const struct jl_typemap_info cfunction_cache = {
-    1, &jl_voidpointer_type
+    1, (jl_datatype_t**)&jl_array_any_type
 };
 
-// Get the LLVM Function* for the C-callable entry point for a certain function
-// and argument types.
-// here argt does not include the leading function type argument
-static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_tupletype_t *argt)
+jl_array_t *jl_cfunction_list;
+
+static Function *jl_cfunction_cache(jl_value_t *ft, jl_value_t *declrt, jl_tupletype_t *argt)
 {
     // Assumes the codegen lock is acquired. The caller is responsible for that.
+    jl_value_t *sigt = NULL; // dispatch sig: type signature (argt) with Ref{} annotations removed and ft added
+    JL_GC_PUSH2(&sigt, &ft);
 
     // validate and unpack the arguments
+    JL_TYPECHK(cfunction, type, (jl_value_t*)ft);
     JL_TYPECHK(cfunction, type, declrt);
-    JL_TYPECHK(cfunction, type, (jl_value_t*)argt);
-    if (!jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(ff)))
-        jl_error("closures are not yet c-callable");
+    if (!jl_is_tuple_type(argt))
+        jl_type_error("cfunction", (jl_value_t*)jl_anytuple_type_type, (jl_value_t*)argt);
+
+    // check the cache structure
+    // this has three levels (for the 3 parameters above)
+    // first split on `ft` using a simple eqtable
+    // then use the typemap to split on argt
+    // and finally, pick declrt from the pair-list
+    union jl_typemap_t cache_l2 = { NULL };
+    // cache_l2.unknown = NULL;
+    jl_typemap_entry_t *cache_l3 = NULL;
+    if (!jl_cfunction_list) {
+        jl_cfunction_list = jl_alloc_vec_any(16);
+    }
+    else {
+        cache_l2.unknown = jl_eqtable_get(jl_cfunction_list, ft, NULL);
+        if (cache_l2.unknown) {
+            cache_l3 = jl_typemap_assoc_by_type(cache_l2, (jl_value_t*)argt, NULL,
+                /*subtype*/0, /*offs*/0, /*world*/1, /*max_world_mask*/0);
+            if (cache_l3) {
+                jl_svec_t *sf = (jl_svec_t*)cache_l3->func.value;
+                size_t i, l = jl_svec_len(sf);
+                for (i = 0; i < l; i += 2) {
+                    jl_value_t *ti = jl_svecref(sf, i);
+                    if (jl_egal(ti, declrt)) {
+                        JL_GC_POP();
+                        return (Function*)jl_unbox_voidpointer(jl_svecref(sf, i + 1));
+                    }
+                }
+            }
+        }
+    }
 
-    size_t i, nargs = jl_nparams(argt);
-    jl_value_t *sigt = NULL; // type signature with Ref{} annotations removed
-    jl_value_t *cfunc_sig = NULL; // type signature of the call to cfunction (for caching)
-    JL_GC_PUSH2(&sigt, &cfunc_sig);
-    sigt = (jl_value_t*)jl_alloc_svec(nargs + 1);
-    cfunc_sig = (jl_value_t*)jl_alloc_svec(nargs + 2);
+    if (cache_l3 == NULL) {
+        union jl_typemap_t insert = cache_l2;
+        if (!insert.unknown)
+            insert.unknown = jl_nothing;
+        cache_l3 = jl_typemap_insert(&insert, (jl_value_t*)insert.unknown, (jl_tupletype_t*)argt,
+            NULL, jl_emptysvec, (jl_value_t*)jl_emptysvec, /*offs*/0, &cfunction_cache, 1, ~(size_t)0, NULL);
+        if (insert.unknown != cache_l2.unknown)
+            jl_cfunction_list = jl_eqtable_put(jl_cfunction_list, ft, insert.unknown, NULL);
+    }
+
+    // try to avoid needing a trampoline,
+    // if we know that `ft` is some sort of
+    // guaranteed singleton type
+    jl_value_t *ff = NULL;
+    if (jl_is_datatype(ft))
+        ff = ((jl_datatype_t*)ft)->instance;
+    if (jl_is_type_type(ft)) {
+        jl_value_t *tp0 = jl_tparam0(ft);
+        if (jl_is_concrete_type(tp0))
+            ff = tp0;
+    }
+    if (!ff)
+        jl_error("cfunction: function closures not yet supported");
 
+    // compute / validate return type
     jl_value_t *crt = declrt;
-    jl_svecset(cfunc_sig, nargs + 1, declrt);
     if (jl_is_abstract_ref_type(declrt)) {
         declrt = jl_tparam0(declrt);
         if (jl_is_typevar(declrt))
             jl_error("cfunction: return type Ref should have an element type, not Ref{<:T}");
         if (declrt == (jl_value_t*)jl_any_type)
             jl_error("cfunction: return type Ref{Any} is invalid. Use Any or Ptr{Any} instead.");
-        if (!jl_is_concrete_type(declrt))
-            jl_svecset(cfunc_sig, nargs + 1, declrt); // Ref{Abstract} is the same calling convention as Abstract
         crt = (jl_value_t*)jl_any_type;
     }
+    bool toboxed;
+    Type *lcrt = julia_struct_to_llvm(crt, NULL, &toboxed);
+    if (lcrt == NULL)
+        jl_error("cfunction: return type doesn't correspond to a C type");
+    else if (toboxed)
+        lcrt = T_prjlvalue;
 
-    if (jl_is_type(ff))
-        jl_svecset(sigt, 0, jl_wrap_Type(ff));
-    else
-        jl_svecset(sigt, 0, jl_typeof(ff));
-    jl_svecset(cfunc_sig, 0, jl_svecref(sigt, 0));
+    // compute / validate method signature
+    size_t i, nargs = jl_nparams(argt);
+    sigt = (jl_value_t*)jl_alloc_svec(nargs + 1);
+    jl_svecset(sigt, 0, ft);
     for (i = 0; i < nargs; i++) {
         jl_value_t *ati = jl_tparam(argt, i);
-        jl_svecset(cfunc_sig, i + 1, ati);
         if (jl_is_abstract_ref_type(ati)) {
             ati = jl_tparam0(ati);
             if (jl_is_typevar(ati))
                 jl_error("cfunction: argument type Ref should have an element type, not Ref{<:T}");
-            if (ati != (jl_value_t*)jl_any_type && !jl_is_concrete_type(ati))
-                jl_svecset(cfunc_sig, i + 1, ati); // Ref{Abstract} is the same calling convention as Abstract
         }
         if (jl_is_pointer(ati) && jl_is_typevar(jl_tparam0(ati)))
             jl_error("cfunction: argument type Ptr should have an element type, Ptr{<:T}");
         jl_svecset(sigt, i + 1, ati);
     }
     sigt = (jl_value_t*)jl_apply_tuple_type((jl_svec_t*)sigt);
-    cfunc_sig = (jl_value_t*)jl_apply_tuple_type((jl_svec_t*)cfunc_sig);
-
-    // check the cache
-    jl_typemap_entry_t *sf = NULL;
-    if (jl_cfunction_list.unknown != jl_nothing) {
-        sf = jl_typemap_assoc_by_type(jl_cfunction_list, cfunc_sig, NULL, /*subtype*/0, /*offs*/0, /*world*/1, /*max_world_mask*/0);
-        if (sf) {
-            jl_value_t *v = sf->func.value;
-            if (v) {
-                if (jl_is_svec(v))
-                    v = jl_svecref(v, 0);
-                Function *f = (Function*)jl_unbox_voidpointer(v);
-                JL_GC_POP();
-                return f;
-            }
-        }
-    }
-    if (sf == NULL) {
-        sf = jl_typemap_insert(&jl_cfunction_list, (jl_value_t*)jl_cfunction_list.unknown, (jl_tupletype_t*)cfunc_sig,
-            NULL, jl_emptysvec, NULL, /*offs*/0, &cfunction_cache, 1, ~(size_t)0, NULL);
-    }
 
-    bool toboxed;
-    Type *lcrt = julia_struct_to_llvm(crt, NULL, &toboxed);
-    if (lcrt == NULL)
-        jl_error("cfunction: return type doesn't correspond to a C type");
-    else if (toboxed)
-        lcrt = T_prjlvalue;
+    // emit cfunction (trampoline)
     jl_value_t *err;
     { // scope block for sig
         function_sig_t sig("cfunction", lcrt, crt, toboxed,
@@ -4616,7 +4662,7 @@ static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_t
             err = NULL;
         }
         else {
-            Function *f = gen_cfun_wrapper(sig, ff, crt, (jl_tupletype_t*)argt, sf, declrt, (jl_tupletype_t*)sigt);
+            auto f = gen_cfun_wrapper(sig, ff, cache_l3, declrt, (jl_tupletype_t*)sigt);
             JL_GC_POP();
             return f;
         }

src/gc.c

@@ -2266,7 +2266,8 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp)
         gc_mark_queue_obj(gc_cache, sp, jl_an_empty_vec_any);
     if (jl_module_init_order != NULL)
         gc_mark_queue_obj(gc_cache, sp, jl_module_init_order);
-    gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list.unknown);
+    if (jl_cfunction_list != NULL)
+        gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list);
     gc_mark_queue_obj(gc_cache, sp, jl_anytuple_type_type);
     gc_mark_queue_obj(gc_cache, sp, jl_ANY_flag);
     for (size_t i = 0; i < N_CALL_CACHE; i++)

src/jltypes.c

@@ -129,7 +129,6 @@ jl_value_t *jl_interrupt_exception;
 jl_datatype_t *jl_boundserror_type;
 jl_value_t *jl_memory_exception;
 jl_value_t *jl_readonlymemory_exception;
-union jl_typemap_t jl_cfunction_list;
 
 jl_cgparams_t jl_default_cgparams = {1, 1, 1, 1, 0, NULL, NULL, NULL};
 
@@ -2217,8 +2216,6 @@ void jl_init_types(void)
     jl_compute_field_offsets(jl_unionall_type);
     jl_compute_field_offsets(jl_simplevector_type);
     jl_compute_field_offsets(jl_sym_type);
-
-    jl_cfunction_list.unknown = jl_nothing;
 }
 
 #ifdef __cplusplus

src/julia_internal.h

@@ -531,7 +531,7 @@ jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims,
                                              int isunboxed, int elsz);
 void jl_module_run_initializer(jl_module_t *m);
 extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
-extern union jl_typemap_t jl_cfunction_list;
+extern jl_array_t *jl_cfunction_list JL_GLOBALLY_ROOTED;
 
 #ifdef JL_USE_INTEL_JITEVENTS
 extern char jl_using_intel_jitevents;