Skip to content

Commit

Permalink
cfunction: redesign cache support
Browse files Browse the repository at this point in the history 
Provides a simpler fast path (no allocations and rebuilding type vectors
required to search the cache).
  • Loading branch information
@vtjnash
vtjnash committed Mar 8, 2018
1 parent caa4824 commit 7934810
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 70 deletions.
6 changes: 3 additions & 3 deletions src/ccall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1774,13 +1774,13 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
else if (is_libjulia_func(jl_function_ptr)) {
assert(!isVa && !llvmcall && nargt == 3);
assert(lrt == T_size);
jl_value_t *f = argv[0].constant;
jl_value_t *ft = argv[0].typ;
jl_value_t *frt = argv[1].constant;
if (!frt) {
if (jl_is_type_type(argv[1].typ) && !jl_has_free_typevars(argv[1].typ))
frt = jl_tparam0(argv[1].typ);
}
if (f && frt) {
if (ft != jl_bottom_type && frt) {
jl_value_t *fargt = argv[2].constant;;
JL_GC_PUSH1(&fargt);
if (!fargt && jl_is_type_type(argv[2].typ)) {
Expand All @@ -1794,7 +1794,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
if (fargt && jl_is_tuple_type(fargt)) {
Value *llvmf = NULL;
JL_TRY {
llvmf = jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt);
llvmf = jl_cfunction_cache(ft, frt, (jl_tupletype_t*)fargt);
}
JL_CATCH {
llvmf = NULL;
Expand Down
170 changes: 108 additions & 62 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1414,7 +1414,20 @@ jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t *li, const char *F, size
return fptr;
}

static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt);
static Function *jl_cfunction_cache(jl_value_t *ft, jl_value_t *declrt, jl_tupletype_t *argt);

// Get a pointer to the cache for the C-callable entry point for the given argument types.
// here argt does not include the leading function type argument
static Function *jl_cfunction_object(jl_value_t *f, jl_value_t *declrt, jl_tupletype_t *argt)
{
jl_value_t *ft;
if (jl_is_type(f))
ft = (jl_value_t*)jl_wrap_Type(f);
else
ft = jl_typeof(f);
return jl_cfunction_cache(ft, declrt, argt);
}

// get the address of a C-callable entry point for a function
extern "C" JL_DLLEXPORT
void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt)
Expand Down Expand Up @@ -1443,7 +1456,6 @@ void *jl_function_ptr_by_llvm_name(char *name) {
extern "C" JL_DLLEXPORT
void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name)
{
assert(jl_is_tuple_type(argt));
JL_LOCK(&codegen_lock);
Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt);
// force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation)
Expand Down Expand Up @@ -4168,9 +4180,9 @@ static void emit_cfunc_invalidate(
}
}

static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
jl_value_t *jlrettype, jl_tupletype_t *argt,
jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt)
static Function*
gen_cfun_wrapper(const function_sig_t &sig, jl_value_t *ff,
jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt)
{
// Generate a c-callable wrapper
size_t nargs = sig.nargs;
Expand Down Expand Up @@ -4221,8 +4233,19 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
#endif
Function *cw_proto = function_proto(cw);
// Save the Function object reference
sf->func.value = jl_box_voidpointer((void*)cw_proto);
jl_gc_wb(sf, sf->func.value);
{
jl_value_t *oldsf = sf->func.value;
size_t i, oldlen = jl_svec_len(oldsf);
jl_value_t *newsf = (jl_value_t*)jl_alloc_svec(oldlen + 2);
JL_GC_PUSH1(&newsf);
jl_svecset(newsf, 0, sig.rt);
jl_svecset(newsf, 1, jl_box_voidpointer((void*)cw_proto));
for (i = 0; i < oldlen; i++)
jl_svecset(newsf, i + 2, jl_svecref(oldsf, i));
sf->func.value = newsf;
jl_gc_wb(sf, sf->func.value);
JL_GC_POP();
}

jl_codectx_t ctx(jl_LLVMContext);
ctx.f = cw;
Expand Down Expand Up @@ -4266,10 +4289,11 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
Function::arg_iterator AI = cw->arg_begin();
Value *sretPtr = sig.sret ? &*AI++ : NULL;
jl_cgval_t *inputargs = (jl_cgval_t*)alloca(sizeof(jl_cgval_t) * (nargs + 1));
inputargs[0] = mark_julia_const(ff); // we need to pass the function object even if (even though) it is a ghost
// we need to pass the function object even if (even though) it is a singleton
inputargs[0] = mark_julia_const(ff);
for (size_t i = 0; i < nargs; ++i, ++AI) {
Value *val = &*AI;
jl_value_t *jargty = jl_nth_slot_type((jl_value_t*)argt, i);
jl_value_t *jargty = jl_svecref(sig.at, i);
// figure out how to unpack this type
jl_cgval_t &inputarg = inputargs[i + 1];
if (jl_is_abstract_ref_type(jargty)) {
Expand Down Expand Up @@ -4521,90 +4545,112 @@ static Function *gen_cfun_wrapper(const function_sig_t &sig, jl_function_t *ff,
}

const struct jl_typemap_info cfunction_cache = {
1, &jl_voidpointer_type
1, (jl_datatype_t**)&jl_array_any_type
};

// Get the LLVM Function* for the C-callable entry point for a certain function
// and argument types.
// here argt does not include the leading function type argument
static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_tupletype_t *argt)
jl_array_t *jl_cfunction_list;

static Function *jl_cfunction_cache(jl_value_t *ft, jl_value_t *declrt, jl_tupletype_t *argt)
{
// Assumes the codegen lock is acquired. The caller is responsible for that.
jl_value_t *sigt = NULL; // dispatch sig: type signature (argt) with Ref{} annotations removed and ft added
JL_GC_PUSH2(&sigt, &ft);

// validate and unpack the arguments
JL_TYPECHK(cfunction, type, (jl_value_t*)ft);
JL_TYPECHK(cfunction, type, declrt);
JL_TYPECHK(cfunction, type, (jl_value_t*)argt);
if (!jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(ff)))
jl_error("closures are not yet c-callable");
if (!jl_is_tuple_type(argt))
jl_type_error("cfunction", (jl_value_t*)jl_anytuple_type_type, (jl_value_t*)argt);

// check the cache structure
// this has three levels (for the 3 parameters above)
// first split on `ft` using a simple eqtable
// then use the typemap to split on argt
// and finally, pick declrt from the pair-list
union jl_typemap_t cache_l2 = { NULL };
// cache_l2.unknown = NULL;
jl_typemap_entry_t *cache_l3 = NULL;
if (!jl_cfunction_list) {
jl_cfunction_list = jl_alloc_vec_any(16);
}
else {
cache_l2.unknown = jl_eqtable_get(jl_cfunction_list, ft, NULL);
if (cache_l2.unknown) {
cache_l3 = jl_typemap_assoc_by_type(cache_l2, (jl_value_t*)argt, NULL,
/*subtype*/0, /*offs*/0, /*world*/1, /*max_world_mask*/0);
if (cache_l3) {
jl_svec_t *sf = (jl_svec_t*)cache_l3->func.value;
size_t i, l = jl_svec_len(sf);
for (i = 0; i < l; i += 2) {
jl_value_t *ti = jl_svecref(sf, i);
if (jl_egal(ti, declrt)) {
JL_GC_POP();
return (Function*)jl_unbox_voidpointer(jl_svecref(sf, i + 1));
}
}
}
}
}

size_t i, nargs = jl_nparams(argt);
jl_value_t *sigt = NULL; // type signature with Ref{} annotations removed
jl_value_t *cfunc_sig = NULL; // type signature of the call to cfunction (for caching)
JL_GC_PUSH2(&sigt, &cfunc_sig);
sigt = (jl_value_t*)jl_alloc_svec(nargs + 1);
cfunc_sig = (jl_value_t*)jl_alloc_svec(nargs + 2);
if (cache_l3 == NULL) {
union jl_typemap_t insert = cache_l2;
if (!insert.unknown)
insert.unknown = jl_nothing;
cache_l3 = jl_typemap_insert(&insert, (jl_value_t*)insert.unknown, (jl_tupletype_t*)argt,
NULL, jl_emptysvec, (jl_value_t*)jl_emptysvec, /*offs*/0, &cfunction_cache, 1, ~(size_t)0, NULL);
if (insert.unknown != cache_l2.unknown)
jl_cfunction_list = jl_eqtable_put(jl_cfunction_list, ft, insert.unknown, NULL);
}

// try to avoid needing a trampoline,
// if we know that `ft` is some sort of
// guaranteed singleton type
jl_value_t *ff = NULL;
if (jl_is_datatype(ft))
ff = ((jl_datatype_t*)ft)->instance;
if (jl_is_type_type(ft)) {
jl_value_t *tp0 = jl_tparam0(ft);
if (jl_is_concrete_type(tp0))
ff = tp0;
}
if (!ff)
jl_error("cfunction: function closures not yet supported");

// compute / validate return type
jl_value_t *crt = declrt;
jl_svecset(cfunc_sig, nargs + 1, declrt);
if (jl_is_abstract_ref_type(declrt)) {
declrt = jl_tparam0(declrt);
if (jl_is_typevar(declrt))
jl_error("cfunction: return type Ref should have an element type, not Ref{<:T}");
if (declrt == (jl_value_t*)jl_any_type)
jl_error("cfunction: return type Ref{Any} is invalid. Use Any or Ptr{Any} instead.");
if (!jl_is_concrete_type(declrt))
jl_svecset(cfunc_sig, nargs + 1, declrt); // Ref{Abstract} is the same calling convention as Abstract
crt = (jl_value_t*)jl_any_type;
}
bool toboxed;
Type *lcrt = julia_struct_to_llvm(crt, NULL, &toboxed);
if (lcrt == NULL)
jl_error("cfunction: return type doesn't correspond to a C type");
else if (toboxed)
lcrt = T_prjlvalue;

if (jl_is_type(ff))
jl_svecset(sigt, 0, jl_wrap_Type(ff));
else
jl_svecset(sigt, 0, jl_typeof(ff));
jl_svecset(cfunc_sig, 0, jl_svecref(sigt, 0));
// compute / validate method signature
size_t i, nargs = jl_nparams(argt);
sigt = (jl_value_t*)jl_alloc_svec(nargs + 1);
jl_svecset(sigt, 0, ft);
for (i = 0; i < nargs; i++) {
jl_value_t *ati = jl_tparam(argt, i);
jl_svecset(cfunc_sig, i + 1, ati);
if (jl_is_abstract_ref_type(ati)) {
ati = jl_tparam0(ati);
if (jl_is_typevar(ati))
jl_error("cfunction: argument type Ref should have an element type, not Ref{<:T}");
if (ati != (jl_value_t*)jl_any_type && !jl_is_concrete_type(ati))
jl_svecset(cfunc_sig, i + 1, ati); // Ref{Abstract} is the same calling convention as Abstract
}
if (jl_is_pointer(ati) && jl_is_typevar(jl_tparam0(ati)))
jl_error("cfunction: argument type Ptr should have an element type, Ptr{<:T}");
jl_svecset(sigt, i + 1, ati);
}
sigt = (jl_value_t*)jl_apply_tuple_type((jl_svec_t*)sigt);
cfunc_sig = (jl_value_t*)jl_apply_tuple_type((jl_svec_t*)cfunc_sig);

// check the cache
jl_typemap_entry_t *sf = NULL;
if (jl_cfunction_list.unknown != jl_nothing) {
sf = jl_typemap_assoc_by_type(jl_cfunction_list, cfunc_sig, NULL, /*subtype*/0, /*offs*/0, /*world*/1, /*max_world_mask*/0);
if (sf) {
jl_value_t *v = sf->func.value;
if (v) {
if (jl_is_svec(v))
v = jl_svecref(v, 0);
Function *f = (Function*)jl_unbox_voidpointer(v);
JL_GC_POP();
return f;
}
}
}
if (sf == NULL) {
sf = jl_typemap_insert(&jl_cfunction_list, (jl_value_t*)jl_cfunction_list.unknown, (jl_tupletype_t*)cfunc_sig,
NULL, jl_emptysvec, NULL, /*offs*/0, &cfunction_cache, 1, ~(size_t)0, NULL);
}

bool toboxed;
Type *lcrt = julia_struct_to_llvm(crt, NULL, &toboxed);
if (lcrt == NULL)
jl_error("cfunction: return type doesn't correspond to a C type");
else if (toboxed)
lcrt = T_prjlvalue;
// emit cfunction (trampoline)
jl_value_t *err;
{ // scope block for sig
function_sig_t sig("cfunction", lcrt, crt, toboxed,
Expand All @@ -4616,7 +4662,7 @@ static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_t
err = NULL;
}
else {
Function *f = gen_cfun_wrapper(sig, ff, crt, (jl_tupletype_t*)argt, sf, declrt, (jl_tupletype_t*)sigt);
auto f = gen_cfun_wrapper(sig, ff, cache_l3, declrt, (jl_tupletype_t*)sigt);
JL_GC_POP();
return f;
}
Expand Down
3 changes: 2 additions & 1 deletion src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2266,7 +2266,8 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp)
gc_mark_queue_obj(gc_cache, sp, jl_an_empty_vec_any);
if (jl_module_init_order != NULL)
gc_mark_queue_obj(gc_cache, sp, jl_module_init_order);
gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list.unknown);
if (jl_cfunction_list != NULL)
gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list);
gc_mark_queue_obj(gc_cache, sp, jl_anytuple_type_type);
gc_mark_queue_obj(gc_cache, sp, jl_ANY_flag);
for (size_t i = 0; i < N_CALL_CACHE; i++)
Expand Down
3 changes: 0 additions & 3 deletions src/jltypes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ jl_value_t *jl_interrupt_exception;
jl_datatype_t *jl_boundserror_type;
jl_value_t *jl_memory_exception;
jl_value_t *jl_readonlymemory_exception;
union jl_typemap_t jl_cfunction_list;

jl_cgparams_t jl_default_cgparams = {1, 1, 1, 1, 0, NULL, NULL, NULL};

Expand Down Expand Up @@ -2217,8 +2216,6 @@ void jl_init_types(void)
jl_compute_field_offsets(jl_unionall_type);
jl_compute_field_offsets(jl_simplevector_type);
jl_compute_field_offsets(jl_sym_type);

jl_cfunction_list.unknown = jl_nothing;
}

#ifdef __cplusplus
Expand Down
2 changes: 1 addition & 1 deletion src/julia_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,7 @@ jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims,
int isunboxed, int elsz);
void jl_module_run_initializer(jl_module_t *m);
extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
extern union jl_typemap_t jl_cfunction_list;
extern jl_array_t *jl_cfunction_list JL_GLOBALLY_ROOTED;

#ifdef JL_USE_INTEL_JITEVENTS
extern char jl_using_intel_jitevents;
Expand Down

0 comments on commit 7934810

Please sign in to comment.