diff --git a/src/anticodegen.c b/src/anticodegen.c index 1b157af267b1c..eb7f4be526175 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -27,7 +27,6 @@ JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *Bytes, uint64_t int32_t jl_assign_functionID(const char *fname) UNAVAILABLE void jl_init_codegen(void) { } -void jl_fptr_to_llvm(void *fptr, jl_method_instance_t *lam, int specsig) { } int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline) { diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 65be12c6bbb3c..c2b0d64318b44 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -536,7 +536,7 @@ void jl_dump_native(void *native_code, // reflect the address of the jl_RTLD_DEFAULT_handle variable // back to the caller, so that we can check for consistency issues - GlobalValue *jlRTLD_DEFAULT_var = data->M->getNamedValue("jl_RTLD_DEFAULT_handle"); + GlobalValue *jlRTLD_DEFAULT_var = jl_emit_RTLD_DEFAULT_var(data->M.get()); addComdat(new GlobalVariable(*data->M, jlRTLD_DEFAULT_var->getType(), true, diff --git a/src/ccall.cpp b/src/ccall.cpp index 634c733bf4a4c..9170fcabfdb2d 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -3,6 +3,12 @@ // --- the ccall, cglobal, and llvm intrinsics --- #include "llvm/Support/Path.h" // for llvm::sys::path +// somewhat unusual variable, in that aotcompile wants to get the address of this for a sanity check +GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M) +{ + return prepare_global_in(M, jlRTLD_DEFAULT_var); +} + // Find or create the GVs for the library and symbol lookup. // Return `runtime_lib` (whether the library name is a string) // The `lib` and `sym` GV returned may not be in the current module. @@ -15,17 +21,17 @@ static bool runtime_sym_gvs(jl_codegen_params_t &emission_context, const char *f jl_codegen_params_t::SymMapGV *symMap; #ifdef _OS_WINDOWS_ if ((intptr_t)f_lib == 1) { - libptrgv = jlexe_var; + libptrgv = prepare_global_in(M, jlexe_var); symMap = &emission_context.symMapExe; } else if ((intptr_t)f_lib == 2) { - libptrgv = jldll_var; + libptrgv = prepare_global_in(M, jldll_var); symMap = &emission_context.symMapDl; } else #endif if (f_lib == NULL) { - libptrgv = jlRTLD_DEFAULT_var; + libptrgv = jl_emit_RTLD_DEFAULT_var(M); symMap = &emission_context.symMapDefault; } else { @@ -636,6 +642,39 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg return mark_julia_type(ctx, res, false, rt); } +static Function *llvmcall_proto(Function *F, Module *M = nullptr) +{ + // Copy the declaration characteristics of the Function (not the body) + Function *NewF = Function::Create(F->getFunctionType(), + Function::ExternalLinkage, + F->getName(), + M); + + // Declarations are not allowed to have personality routines, but + // copyAttributesFrom sets them anyway. Temporarily unset the personality + // routine from `F`, since copying it and then resetting is more expensive + // as well as introducing an extra use from this unowned function, which + // can cause crashes in the LLVMContext's global destructor. + llvm::Constant *OldPersonalityFn = nullptr; + if (F->hasPersonalityFn()) { + OldPersonalityFn = F->getPersonalityFn(); + F->setPersonalityFn(nullptr); + } + + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + + if (OldPersonalityFn) + F->setPersonalityFn(OldPersonalityFn); + + // DLLImport only needs to be set for the shadow module + // it just gets annoying in the JIT + NewF->setDLLStorageClass(GlobalValue::DefaultStorageClass); + + return NewF; +} + class FunctionMover final : public ValueMaterializer { public: @@ -699,7 +738,7 @@ class FunctionMover final : public ValueMaterializer { Function *NewF = destModule->getFunction(F->getName()); if (!NewF) { - NewF = function_proto(F); + NewF = llvmcall_proto(F); NewF->setComdat(nullptr); destModule->getFunctionList().push_back(NewF); } @@ -776,6 +815,15 @@ class FunctionMover final : public ValueMaterializer }; }; +static Function *prepare_llvmcall(Module *M, Function *Callee) +{ + GlobalValue *local = M->getNamedValue(Callee->getName()); + if (!local) + local = llvmcall_proto(Callee, M); + return cast(local); +} + + // llvmcall(ir, (rettypes...), (argtypes...), args...) static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) { @@ -954,7 +1002,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar std::stringstream name; name << "jl_llvmcall" << llvmcallnumbering++; f->setName(name.str()); - f = cast(prepare_call(function_proto(f)).getCallee()); + f = prepare_llvmcall(jl_Module, llvmcall_proto(f)); } else { f->setLinkage(GlobalValue::LinkOnceODRLinkage); @@ -1862,9 +1910,9 @@ jl_cgval_t function_sig_t::emit_a_ccall( OperandBundleDef OpBundle("jl_roots", gc_uses); // the actual call - Value *ret = ctx.builder.CreateCall(prepare_call(llvmf), - ArrayRef(&argvals[0], nccallargs + sret), - ArrayRef(&OpBundle, gc_uses.empty() ? 0 : 1)); + CallInst *ret = ctx.builder.CreateCall(functype, llvmf, + ArrayRef(&argvals[0], nccallargs + sret), + ArrayRef(&OpBundle, gc_uses.empty() ? 0 : 1)); ((CallInst*)ret)->setAttributes(attributes); if (cc != CallingConv::C) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index abcd5cbb40e20..64c7056cb359a 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -10,54 +10,6 @@ static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) return load_or_store; } -static Function *function_proto(Function *F, Module *M = nullptr) -{ - // Copy the declaration characteristics of the Function (not the body) - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - M); - - // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway. Temporarily unset the personality - // routine from `F`, since copying it and then resetting is more expensive - // as well as introducing an extra use from this unowned function, which - // can cause crashes in the LLVMContext's global destructor. - llvm::Constant *OldPersonalityFn = nullptr; - if (F->hasPersonalityFn()) { - OldPersonalityFn = F->getPersonalityFn(); - F->setPersonalityFn(nullptr); - } - - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); - - if (OldPersonalityFn) - F->setPersonalityFn(OldPersonalityFn); - - // DLLImport only needs to be set for the shadow module - // it just gets annoying in the JIT - NewF->setDLLStorageClass(GlobalValue::DefaultStorageClass); - - return NewF; -} - -#define prepare_call(Callee) prepare_call_in(jl_Module, (Callee)) -static FunctionCallee prepare_call_in(Module *M, Value *Callee) -{ - if (Function *F = dyn_cast(Callee)) { - GlobalValue *local = M->getNamedValue(Callee->getName()); - if (!local) { - local = function_proto(F, M); - } - Callee = local; - } - FunctionType *FnTy = cast( - Callee->getType()->getPointerElementType()); - return {FnTy, Callee}; -} - // Take an arbitrary untracked value and make it gc-tracked static Value *maybe_decay_untracked(IRBuilder<> &irbuilder, Value *V) { @@ -256,8 +208,9 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) Type *T = PointerType::get(T_jlvalue, AddressSpace::Derived); if (V->getType() != T) V = ctx.builder.CreateBitCast(V, T); - CallInst *Call = ctx.builder.CreateCall(prepare_call(pointer_from_objref_func), V); - Call->setAttributes(pointer_from_objref_func->getAttributes()); + Function *F = prepare_call(pointer_from_objref_func); + CallInst *Call = ctx.builder.CreateCall(F, V); + Call->setAttributes(F->getAttributes()); return Call; } @@ -332,7 +285,7 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, j return julia_pgv(ctx, fullname, addr); } -static GlobalVariable *julia_const_gv(jl_value_t *val); +static JuliaVariable *julia_const_gv(jl_value_t *val); static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) { // emit a pointer to a jl_value_t* which will allow it to be valid across reloading code @@ -345,7 +298,7 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); return gv; } - if (GlobalVariable *gv = julia_const_gv(p)) { + if (JuliaVariable *gv = julia_const_gv(p)) { // if this is a known special object, use the existing GlobalValue return prepare_global_in(jl_Module, gv); } @@ -2225,10 +2178,11 @@ static jl_value_t *static_constant_instance(Constant *constant, jl_value_t *jt) return obj; } -static Value *call_with_attrs(jl_codectx_t &ctx, Function *func, Value *v) +static Value *call_with_attrs(jl_codectx_t &ctx, JuliaFunction *intr, Value *v) { - CallInst *Call = ctx.builder.CreateCall(prepare_call(func), v); - Call->setAttributes(func->getAttributes()); + Function *F = prepare_call(intr); + CallInst *Call = ctx.builder.CreateCall(F, v); + Call->setAttributes(F->getAttributes()); return Call; } @@ -2607,10 +2561,9 @@ static void emit_cpointercheck(jl_codectx_t &ctx, const jl_cgval_t &x, const std static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt) { Value *ptls_ptr = emit_bitcast(ctx, ctx.ptlsStates, T_pint8); - auto call = ctx.builder.CreateCall(prepare_call(jl_alloc_obj_func), - {ptls_ptr, ConstantInt::get(T_size, static_size), - maybe_decay_untracked(jt)}); - call->setAttributes(jl_alloc_obj_func->getAttributes()); + Function *F = prepare_call(jl_alloc_obj_func); + auto call = ctx.builder.CreateCall(F, {ptls_ptr, ConstantInt::get(T_size, static_size), maybe_decay_untracked(jt)}); + call->setAttributes(F->getAttributes()); return call; } @@ -2618,8 +2571,9 @@ static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt) static Value *emit_new_bits(jl_codectx_t &ctx, Value *jt, Value *pval) { pval = ctx.builder.CreateBitCast(pval, T_pint8); - auto call = ctx.builder.CreateCall(prepare_call(jl_newbits_func), { jt, pval }); - call->setAttributes(jl_newbits_func->getAttributes()); + Function *F = prepare_call(jl_newbits_func); + auto call = ctx.builder.CreateCall(F, { jt, pval }); + call->setAttributes(F->getAttributes()); return call; } diff --git a/src/codegen.cpp b/src/codegen.cpp index ff1b10025f7ec..5ef7be0373197 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -156,6 +156,7 @@ TargetMachine *jl_TargetMachine; static DataLayout &jl_data_layout = *(new DataLayout("")); #define jl_Module ctx.f->getParent() #define jl_builderModule(builder) (builder).GetInsertBlock()->getParent()->getParent() +#define prepare_call(Callee) prepare_call_in(jl_Module, (Callee)) // types static Type *T_jlvalue; @@ -272,88 +273,542 @@ static bool is_uniquerep_Type(jl_value_t *t) return jl_is_type_type(t) && type_has_unique_rep(jl_tparam0(t)); } +class jl_codectx_t; +struct JuliaVariable { +public: + StringLiteral name; + bool isconst; + Type *(*_type)(LLVMContext &C); + + JuliaVariable(const JuliaVariable&) = delete; + JuliaVariable(const JuliaVariable&&) = delete; + GlobalVariable *realize(Module *m) { + if (GlobalValue *V = m->getNamedValue(name)) + return cast(V); + return new GlobalVariable(*m, _type(m->getContext()), + isconst, GlobalVariable::ExternalLinkage, + NULL, name); + } + GlobalVariable *realize(jl_codectx_t &ctx); +}; +static inline void add_named_global(JuliaVariable *name, void *addr) +{ + add_named_global(name->name, addr); +} + +struct JuliaFunction { +public: + StringLiteral name; + FunctionType *(*_type)(LLVMContext &C); + AttributeList (*_attrs)(LLVMContext &C); + + JuliaFunction(const JuliaFunction&) = delete; + JuliaFunction(const JuliaFunction&&) = delete; + Function *realize(Module *m) { + if (GlobalValue *V = m->getNamedValue(name)) + return cast(V); + Function *F = Function::Create(_type(m->getContext()), + Function::ExternalLinkage, + name, m); + if (_attrs) + F->setAttributes(_attrs(m->getContext())); + return F; + } + Function *realize(jl_codectx_t &ctx); +}; +template +static inline void add_named_global(JuliaFunction *name, T *addr) +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + add_named_global(name->name, (void*)(uintptr_t)addr); +} +template +static inline void add_named_global(StringRef name, T *addr) +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + add_named_global(name, (void*)(uintptr_t)addr); +} + +AttributeSet Attributes(LLVMContext &C, std::initializer_list attrkinds) +{ + SmallVector attrs(attrkinds.size()); + for (size_t i = 0; i < attrkinds.size(); i++) + attrs[i] = Attribute::get(C, attrkinds.begin()[i]); + return AttributeSet::get(C, makeArrayRef(attrs)); +} + +static Type *get_pjlvalue(LLVMContext &C) { return T_pjlvalue; } + +static FunctionType *get_func_sig(LLVMContext &C) { return jl_func_sig; } + +static AttributeList get_func_attrs(LLVMContext &C) +{ + return AttributeList::get(C, + AttributeSet::get(C, makeArrayRef({Thunk})), + Attributes(C, {Attribute::NonNull}), + None); +} + +static AttributeList get_attrs_noreturn(LLVMContext &C) +{ + return AttributeList::get(C, + Attributes(C, {Attribute::NoReturn}), + AttributeSet(), + None); +} + +static AttributeList get_attrs_sext(LLVMContext &C) +{ + return AttributeList::get(C, + AttributeSet(), + AttributeSet(), + {Attributes(C, {Attribute::SExt})}); +} + +static AttributeList get_attrs_zext(LLVMContext &C) +{ + return AttributeList::get(C, + AttributeSet(), + AttributeSet(), + {Attributes(C, {Attribute::ZExt})}); +} + + // global vars -static GlobalVariable *jlRTLD_DEFAULT_var; +static const auto jlRTLD_DEFAULT_var = new JuliaVariable{ + "jl_RTLD_DEFAULT_handle", + true, + [](LLVMContext &C) { return T_pint8; }, +}; #ifdef _OS_WINDOWS_ -static GlobalVariable *jlexe_var; -static GlobalVariable *jldll_var; +static const auto jlexe_var = new JuliaVariable{ + "jl_exe_handle", + true, + [](LLVMContext &C) { return T_pint8; }, +}; +static const auto jldll_var = new JuliaVariable{ + "jl_dl_handle", + true, + [](LLVMContext &C) { return T_pint8; }, +}; #endif //_OS_WINDOWS_ -static Function *jltls_states_func; +static const auto jlstack_chk_guard_var = new JuliaVariable{ + "__stack_chk_guard", + true, + get_pjlvalue, +}; + +static const auto jlgetworld_global = new JuliaVariable{ + "jl_world_counter", + false, + [](LLVMContext &C) { return (Type*)T_size; }, +}; + +static const auto jltls_states_func = new JuliaFunction{ + "julia.ptls_states", + [](LLVMContext &C) { return FunctionType::get(PointerType::get(T_ppjlvalue, 0), false); }, +}; + // important functions -static Function *jlnew_func; -static Function *jlsplatnew_func; -static Function *jlthrow_func; -static Function *jlerror_func; -static Function *jltypeerror_func; -static Function *jlundefvarerror_func; -static Function *jlboundserror_func; -static Function *jluboundserror_func; -static Function *jlvboundserror_func; -static Function *jlboundserrorv_func; -static Function *jlcheckassign_func; -static Function *jldeclareconst_func; -static Function *jlgetbindingorerror_func; -static Function *jlboundp_func; -static Function *jltopeval_func; -static Function *jlcopyast_func; -static Function *jltuple_func; -static Function *jlnsvec_func; -static Function *jlapplygeneric_func; -static Function *jlinvoke_func; -static Function *jlgetfield_func; -static Function *jlmethod_func; -static Function *jlgenericfunction_func; -static Function *jlenter_func; -static Function *jl_current_exception_func; -static Function *jlleave_func; -static Function *jl_restore_excstack_func; -static Function *jl_excstack_state_func; -static Function *jlegal_func; -static Function *jl_alloc_obj_func; -static Function *jl_newbits_func; -static Function *jl_typeof_func; -static Function *jl_loopinfo_marker_func; -static Function *jl_write_barrier_func; -static Function *jlisa_func; -static Function *jlsubtype_func; -static Function *jlapplytype_func; -static Function *jl_object_id__func; -static Function *setjmp_func; -static Function *memcmp_func; -static Function *box_int8_func; -static Function *box_uint8_func; -static Function *box_int16_func; -static Function *box_uint16_func; -static Function *box_int32_func; -static Function *box_char_func; -static Function *box_uint32_func; -static Function *box_int64_func; -static Function *box_uint64_func; -static Function *box_float32_func; -static Function *box_float64_func; -static Function *box_ssavalue_func; -static Function *expect_func; -static Function *jldlsym_func; -static Function *jltypeassert_func; -//static Function *jlgetnthfield_func; -static Function *jlgetnthfieldchecked_func; -//static Function *jlsetnthfield_func; -static Function *jlgetcfunctiontrampoline_func; -static Function *diff_gc_total_bytes_func; -static Function *sync_gc_total_bytes_func; -static Function *jlarray_data_owner_func; -static GlobalVariable *jlgetworld_global; +// Symbols are not gc-tracked, but we'll treat them as callee rooted anyway, +// because they may come from a gc-rooted location +static const auto jlnew_func = new JuliaFunction{ + "jl_new_structv", + get_func_sig, + get_func_attrs, +}; +static const auto jlsplatnew_func = new JuliaFunction{ + "jl_new_structt", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue, T_prjlvalue}, false); }, + get_func_attrs, +}; +static const auto jlthrow_func = new JuliaFunction{ + "jl_throw", + [](LLVMContext &C) { return FunctionType::get(T_void, + {PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)}, false); }, + get_attrs_noreturn, +}; +static const auto jlerror_func = new JuliaFunction{ + "jl_error", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pint8}, false); }, + get_attrs_noreturn, +}; +static const auto jltypeerror_func = new JuliaFunction{ + "jl_type_error", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pint8, T_prjlvalue, PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)}, false); }, + get_attrs_noreturn, +}; +static const auto jlundefvarerror_func = new JuliaFunction{ + "jl_undefined_var_error", + [](LLVMContext &C) { return FunctionType::get(T_void, + {PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)}, false); }, + get_attrs_noreturn, +}; +static const auto jlboundserrorv_func = new JuliaFunction{ + "jl_bounds_error_ints", + [](LLVMContext &C) { return FunctionType::get(T_void, + {PointerType::get(T_jlvalue, AddressSpace::CalleeRooted), T_psize, T_size}, false); }, + get_attrs_noreturn, +}; +static const auto jlboundserror_func = new JuliaFunction{ + "jl_bounds_error_int", + [](LLVMContext &C) { return FunctionType::get(T_void, + {PointerType::get(T_jlvalue, AddressSpace::CalleeRooted), T_size}, false); }, + get_attrs_noreturn, +}; +static const auto jlvboundserror_func = new JuliaFunction{ + "jl_bounds_error_tuple_int", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pprjlvalue, T_size, T_size}, false); }, + get_attrs_noreturn, +}; +static const auto jluboundserror_func = new JuliaFunction{ + "jl_bounds_error_unboxed_int", + [](LLVMContext &C) { return FunctionType::get(T_void, + {PointerType::get(T_int8, AddressSpace::Derived), T_pjlvalue, T_size}, false); }, + get_attrs_noreturn, +}; +static const auto jlcheckassign_func = new JuliaFunction{ + "jl_checked_assignment", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pjlvalue, PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)}, false); }, +}; +static const auto jldeclareconst_func = new JuliaFunction{ + "jl_declare_constant", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pjlvalue}, false); }, +}; +static const auto jlgetbindingorerror_func = new JuliaFunction{ + "jl_get_binding_or_error", + [](LLVMContext &C) { return FunctionType::get(T_pjlvalue, + {T_pjlvalue, T_pjlvalue}, false); }, +}; +static const auto jlboundp_func = new JuliaFunction{ + "jl_boundp", + [](LLVMContext &C) { return FunctionType::get(T_int32, + {T_pjlvalue, T_pjlvalue}, false); }, +}; +static const auto jltopeval_func = new JuliaFunction{ + "jl_toplevel_eval", + [](LLVMContext &C) { return FunctionType::get(T_pjlvalue, + {T_pjlvalue, T_pjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +static const auto jlcopyast_func = new JuliaFunction{ + "jl_copy_ast", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +//static const auto jlnsvec_func = new JuliaFunction{ +// "jl_svec", +// [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, +// {T_size}, true); }, +// [](LLVMContext &C) { return AttributeList::get(C, +// AttributeSet(), +// Attributes(C, {Attribute::NonNull}), +// None); }, +//}; +static const auto jlapplygeneric_func = new JuliaFunction{ + "jl_apply_generic", + get_func_sig, + get_func_attrs, +}; +static const auto jlinvoke_func = new JuliaFunction{ + "jl_invoke", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue, T_pprjlvalue, T_uint32, T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + {AttributeSet(), + Attributes(C, {Attribute::ReadOnly, Attribute::NoCapture})}); }, +}; +static const auto jlmethod_func = new JuliaFunction{ + "jl_method_def", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_prjlvalue, T_prjlvalue, T_pjlvalue}, false); }, +}; +static const auto jlgenericfunction_func = new JuliaFunction{ + "jl_generic_function_def", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pprjlvalue, T_pjlvalue, T_pjlvalue}, false); }, +}; +static const auto jlenter_func = new JuliaFunction{ + "jl_enter_handler", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_pint8}, false); }, +}; +static const auto jl_current_exception_func = new JuliaFunction{ + "jl_current_exception", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, false); }, +}; +static const auto jlleave_func = new JuliaFunction{ + "jl_pop_handler", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_int32}, false); }, +}; +static const auto jl_restore_excstack_func = new JuliaFunction{ + "jl_restore_excstack", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_size}, false); }, +}; +static const auto jl_excstack_state_func = new JuliaFunction{ + "jl_excstack_state", + [](LLVMContext &C) { return FunctionType::get(T_size, false); }, +}; +static const auto jlegal_func = new JuliaFunction{ + "jl_egal", + [](LLVMContext &C) { + Type *T = PointerType::get(T_jlvalue, AddressSpace::CalleeRooted); + return FunctionType::get(T_int32, {T, T}, false); }, +}; +static const auto jl_alloc_obj_func = new JuliaFunction{ + "julia.gc_alloc_obj", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_pint8, T_size, T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet::get(C, makeArrayRef({Attribute::getWithAllocSizeArgs(C, 1, None)})), // returns %1 bytes + Attributes(C, {Attribute::NoAlias, Attribute::NonNull}), + None); }, +}; +static const auto jl_newbits_func = new JuliaFunction{ + "jl_new_bits", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue, T_pint8}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NoAlias, Attribute::NonNull}), + None); }, +}; +static const auto jl_typeof_func = new JuliaFunction{ + "julia.typeof", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReadOnly, Attribute::NoUnwind, Attribute::ArgMemOnly, Attribute::NoRecurse}), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +static const auto jl_loopinfo_marker_func = new JuliaFunction{ + "julia.loopinfo_marker", + [](LLVMContext &C) { return FunctionType::get(T_void, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReadOnly, Attribute::NoRecurse, Attribute::InaccessibleMemOnly}), + AttributeSet(), + None); }, +}; +static const auto jl_write_barrier_func = new JuliaFunction{ + "julia.write_barrier", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_prjlvalue}, true); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::NoUnwind, Attribute::NoRecurse, Attribute::InaccessibleMemOnly}), + AttributeSet(), + None); }, +}; +static const auto jlisa_func = new JuliaFunction{ + "jl_isa", + [](LLVMContext &C) { return FunctionType::get(T_int32, + {T_prjlvalue, T_prjlvalue}, false); }, +}; + +static const auto jlsubtype_func = new JuliaFunction{ + "jl_subtype", + [](LLVMContext &C) { return FunctionType::get(T_int32, + {T_prjlvalue, T_prjlvalue}, false); }, +}; +static const auto jlapplytype_func = new JuliaFunction{ + "jl_instantiate_type_in_env", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pprjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +static const auto jl_object_id__func = new JuliaFunction{ + "jl_object_id_", + [](LLVMContext &C) { return FunctionType::get(T_size, + {T_prjlvalue, PointerType::get(T_int8, AddressSpace::Derived)}, false); }, +}; +static const auto setjmp_func = new JuliaFunction{ + jl_setjmp_name, + [](LLVMContext &C) { return FunctionType::get(T_int32, + {T_pint8, +#ifndef _OS_WINDOWS_ + T_int32, +#endif + }, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReturnsTwice}), + AttributeSet(), + None); }, +}; +static const auto memcmp_func = new JuliaFunction{ + "memcmp", + [](LLVMContext &C) { return FunctionType::get(T_int32, + {T_pint8, T_pint8, T_size}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReadOnly, Attribute::NoUnwind, Attribute::ArgMemOnly}), + AttributeSet(), + None); }, + // TODO: inferLibFuncAttributes(*memcmp_func, TLI); +}; +static const auto jldlsym_func = new JuliaFunction{ + "jl_load_and_lookup", + [](LLVMContext &C) { return FunctionType::get(T_pvoidfunc, + {T_pint8, T_pint8, PointerType::get(T_pint8, 0)}, false); }, +}; +static const auto jltypeassert_func = new JuliaFunction{ + "jl_typeassert", + [](LLVMContext &C) { return FunctionType::get(T_void, + {T_prjlvalue, T_prjlvalue}, false); }, +}; +static const auto jlgetnthfieldchecked_func = new JuliaFunction{ + "jl_get_nth_field_checked", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue, T_size}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +static const auto jlgetcfunctiontrampoline_func = new JuliaFunction{ + "jl_get_cfunction_trampoline", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + { + T_prjlvalue, // f (object) + T_pjlvalue, // result + T_pint8, // cache + T_pjlvalue, // fill + FunctionType::get(T_pint8, { T_pint8, T_ppjlvalue }, false)->getPointerTo(), // trampoline + T_pjlvalue, // env + T_pprjlvalue, // vals + }, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +static const auto diff_gc_total_bytes_func = new JuliaFunction{ + "jl_gc_diff_total_bytes", + [](LLVMContext &C) { return FunctionType::get(T_int64, false); }, +}; +static const auto sync_gc_total_bytes_func = new JuliaFunction{ + "jl_gc_sync_total_bytes", + [](LLVMContext &C) { return FunctionType::get(T_int64, + {T_int64}, false); }, +}; +static const auto jlarray_data_owner_func = new JuliaFunction{ + "jl_array_data_owner", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReadOnly, Attribute::NoUnwind}), + Attributes(C, {Attribute::NonNull}), + None); }, +}; +#define BOX_FUNC(ct,rt,at,attrs) \ +static const auto box_##ct##_func = new JuliaFunction{ \ + "jl_box_"#ct, \ + [](LLVMContext &C) { return FunctionType::get(rt, \ + {at}, false); }, \ + attrs, \ +} +BOX_FUNC(int8, T_pjlvalue, T_int8, get_attrs_sext); +BOX_FUNC(uint8, T_pjlvalue, T_int8, get_attrs_zext); +BOX_FUNC(int16, T_prjlvalue, T_int16, get_attrs_sext); +BOX_FUNC(uint16, T_prjlvalue, T_int16, get_attrs_zext); +BOX_FUNC(int32, T_prjlvalue, T_int32, get_attrs_sext); +BOX_FUNC(uint32, T_prjlvalue, T_int32, get_attrs_zext); +BOX_FUNC(int64, T_prjlvalue, T_int64, get_attrs_sext); +BOX_FUNC(uint64, T_prjlvalue, T_int64, get_attrs_zext); +BOX_FUNC(char, T_prjlvalue, T_char, get_attrs_zext); +BOX_FUNC(float32, T_prjlvalue, T_float32, nullptr); +BOX_FUNC(float64, T_prjlvalue, T_float64, nullptr); +BOX_FUNC(ssavalue, T_prjlvalue, T_size, nullptr); +#undef BOX_FUNC + // placeholder functions -static Function *gcroot_flush_func; -static Function *gc_preserve_begin_func; -static Function *gc_preserve_end_func; -static Function *except_enter_func; -static Function *pointer_from_objref_func; +static const auto gcroot_flush_func = new JuliaFunction{ + "julia.gcroot_flush", + [](LLVMContext &C) { return FunctionType::get(T_void, false); }, +}; +static const auto gc_preserve_begin_func = new JuliaFunction{ + "llvm.julia.gc_preserve_begin", + [](LLVMContext &C) { return FunctionType::get(Type::getTokenTy(C), true); }, +}; +static const auto gc_preserve_end_func = new JuliaFunction { + "llvm.julia.gc_preserve_end", + [](LLVMContext &C) { return FunctionType::get(T_void, {Type::getTokenTy(C)}, false); }, +}; +static const auto except_enter_func = new JuliaFunction{ + "julia.except_enter", + [](LLVMContext &C) { return FunctionType::get(T_int32, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet::get(C, makeArrayRef({Attribute::get(C, Attribute::ReturnsTwice)})), + AttributeSet(), + None); }, +}; +static const auto pointer_from_objref_func = new JuliaFunction{ + "julia.pointer_from_objref", + [](LLVMContext &C) { return FunctionType::get(T_pjlvalue, + {PointerType::get(T_jlvalue, AddressSpace::Derived)}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet::get(C, makeArrayRef({Attribute::get(C, Attribute::ReadNone), Attribute::get(C, Attribute::NoUnwind)})), + AttributeSet(), + None); }, +}; + +static const auto jltuple_func = new JuliaFunction{"jl_f_tuple", get_func_sig, get_func_attrs}; +static const auto jlgetfield_func = new JuliaFunction{"jl_f_getfield", get_func_sig, get_func_attrs}; +static const std::map builtin_func_map = { + { &jl_f_is, new JuliaFunction{"jl_f_is", get_func_sig, get_func_attrs} }, + { &jl_f_typeof, new JuliaFunction{"jl_f_typeof", get_func_sig, get_func_attrs} }, + { &jl_f_sizeof, new JuliaFunction{"jl_f_sizeof", get_func_sig, get_func_attrs} }, + { &jl_f_issubtype, new JuliaFunction{"jl_f_issubtype", get_func_sig, get_func_attrs} }, + { &jl_f_isa, new JuliaFunction{"jl_f_isa", get_func_sig, get_func_attrs} }, + { &jl_f_typeassert, new JuliaFunction{"jl_f_typeassert", get_func_sig, get_func_attrs} }, + { &jl_f_ifelse, new JuliaFunction{"jl_f_ifelse", get_func_sig, get_func_attrs} }, + { &jl_f__apply, new JuliaFunction{"jl_f__apply", get_func_sig, get_func_attrs} }, + { &jl_f__apply_iterate, new JuliaFunction{"jl_f__apply_iterate", get_func_sig, get_func_attrs} }, + { &jl_f__apply_pure, new JuliaFunction{"jl_f__apply_pure", get_func_sig, get_func_attrs} }, + { &jl_f__apply_latest, new JuliaFunction{"jl_f__apply_latest", get_func_sig, get_func_attrs} }, + { &jl_f_throw, new JuliaFunction{"jl_f_throw", get_func_sig, get_func_attrs} }, + { &jl_f_tuple, jltuple_func }, + { &jl_f_svec, new JuliaFunction{"jl_f_svec", get_func_sig, get_func_attrs} }, + { &jl_f_applicable, new JuliaFunction{"jl_f_applicable", get_func_sig, get_func_attrs} }, + { &jl_f_invoke, new JuliaFunction{"jl_f_invoke", get_func_sig, get_func_attrs} }, + { &jl_f_invoke_kwsorter, new JuliaFunction{"jl_f_invoke_kwsorter", get_func_sig, get_func_attrs} }, + { &jl_f_isdefined, new JuliaFunction{"jl_f_isdefined", get_func_sig, get_func_attrs} }, + { &jl_f_getfield, jlgetfield_func }, + { &jl_f_setfield, new JuliaFunction{"jl_f_setfield", get_func_sig, get_func_attrs} }, + { &jl_f_fieldtype, new JuliaFunction{"jl_f_fieldtype", get_func_sig, get_func_attrs} }, + { &jl_f_nfields, new JuliaFunction{"jl_f_nfields", get_func_sig, get_func_attrs} }, + { &jl_f__expr, new JuliaFunction{"jl_f__expr", get_func_sig, get_func_attrs} }, + { &jl_f__typevar, new JuliaFunction{"jl_f__typevar", get_func_sig, get_func_attrs} }, + { &jl_f_arrayref, new JuliaFunction{"jl_f_arrayref", get_func_sig, get_func_attrs} }, + { &jl_f_const_arrayref, new JuliaFunction{"jl_f_const_arrayref", get_func_sig, get_func_attrs} }, + { &jl_f_arrayset, new JuliaFunction{"jl_f_arrayset", get_func_sig, get_func_attrs} }, + { &jl_f_arraysize, new JuliaFunction{"jl_f_arraysize", get_func_sig, get_func_attrs} }, + { &jl_f_apply_type, new JuliaFunction{"jl_f_apply_type", get_func_sig, get_func_attrs} }, +}; -static std::map builtin_func_map; // --- code generation --- extern "C" { @@ -590,6 +1045,10 @@ class jl_codectx_t { } }; +GlobalVariable *JuliaVariable::realize(jl_codectx_t &ctx) { + return realize(jl_Module); +} + static Type *julia_type_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, bool *isboxed = NULL); static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype); static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval = -1); @@ -600,13 +1059,45 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i); static Value *emit_condition(jl_codectx_t &ctx, const jl_cgval_t &condV, const std::string &msg); static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0); static void CreateTrap(IRBuilder<> &irbuilder); -static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, +static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, + jl_cgval_t *args, size_t nargs, CallingConv::ID cc); +static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction *theFptr, Value *theF, jl_cgval_t *args, size_t nargs, CallingConv::ID cc); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G); static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store); +static GlobalVariable *prepare_global_in(Module *M, JuliaVariable *G) +{ + return G->realize(M); +} + +static Function *prepare_call_in(Module *M, JuliaFunction *G) +{ + return G->realize(M); +} + +static inline GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G) +{ + if (G->getParent() == M) + return G; + GlobalValue *local = M->getNamedValue(G->getName()); + if (!local) { + // Copy the GlobalVariable, but without the initializer, so it becomes a declaration + GlobalVariable *proto = new GlobalVariable(*M, G->getType()->getElementType(), + G->isConstant(), GlobalVariable::ExternalLinkage, + nullptr, G->getName(), nullptr, G->getThreadLocalMode()); + proto->copyAttributesFrom(G); + // DLLImport only needs to be set for the shadow module + // it just gets annoying in the JIT + proto->setDLLStorageClass(GlobalValue::DefaultStorageClass); + return proto; + } + return cast(local); +} + + // --- convenience functions for tagging llvm values with julia types --- static GlobalVariable *get_pointer_to_constant(Constant *val, StringRef name, Module &M) @@ -2611,7 +3102,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } -static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, +static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) { // emit arguments @@ -2628,12 +3119,17 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, } FunctionType *FTy = FunctionType::get(T_prjlvalue, argsT, false); CallInst *result = ctx.builder.CreateCall(FTy, - ctx.builder.CreateBitCast(prepare_call(theFptr).getCallee(), FTy->getPointerTo()), + ctx.builder.CreateBitCast(theFptr, FTy->getPointerTo()), theArgs); add_return_attr(result, Attribute::NonNull); result->setCallingConv(cc); return result; } +static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction *theFptr, Value *theF, + jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) +{ + return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, cc); +} static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, @@ -2755,16 +3251,13 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, StringRef specFunctionObject, jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) { - auto theFptr = jl_Module->getOrInsertFunction(specFunctionObject, jl_func_sig) + auto theFptr = cast(jl_Module->getOrInsertFunction(specFunctionObject, jl_func_sig) #if JL_LLVM_VERSION >= 90000 - .getCallee(); -#else - ; + .getCallee() #endif - if (auto F = dyn_cast(theFptr->stripPointerCasts())) { - add_return_attr(F, Attribute::NonNull); - F->addFnAttr(Thunk); - } + ); + add_return_attr(theFptr, Attribute::NonNull); + theFptr->addFnAttr(Thunk); Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, JLCALL_F_CC); return mark_julia_type(ctx, ret, true, inferred_retty); } @@ -2850,7 +3343,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) } } if (!handled) { - Value *r = emit_jlcall(ctx, prepare_call(jlinvoke_func).getCallee(), boxed(ctx, lival), argv, nargs, JLCALL_F2_CC); + Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, JLCALL_F2_CC); result = mark_julia_type(ctx, r, true, rt); } if (result.typ == jl_bottom_type) @@ -2888,10 +3381,9 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) } // special case for known builtin not handled by emit_builtin_call - std::map::iterator it = builtin_func_map.find(jl_get_builtin_fptr(f.constant)); + auto it = builtin_func_map.find(jl_get_builtin_fptr(f.constant)); if (it != builtin_func_map.end()) { - Value *theFptr = it->second; - Value *ret = emit_jlcall(ctx, theFptr, maybe_decay_untracked(V_null), &argv[1], nargs - 1, JLCALL_F_CC); + Value *ret = emit_jlcall(ctx, it->second, maybe_decay_untracked(V_null), &argv[1], nargs - 1, JLCALL_F_CC); return mark_julia_type(ctx, ret, true, rt); } } @@ -3909,11 +4401,15 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod ctx.f = f; // for jl_Module BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", f); ctx.builder.SetInsertPoint(b0); - FunctionCallee theFunc; + Function *theFunc; Value *theFarg; if (codeinst->invoke != NULL) { StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->invoke, codeinst); - theFunc = M->getOrInsertFunction(theFptrName, jlinvoke_func->getFunctionType()); + theFunc = cast(M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(jl_LLVMContext)) +#if JL_LLVM_VERSION >= 90000 + .getCallee() +#endif + ); theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst); } else { @@ -3923,7 +4419,7 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod theFarg = maybe_decay_untracked(theFarg); auto args = f->arg_begin(); CallInst *r = ctx.builder.CreateCall(theFunc, { &*args, &*++args, &*++args, theFarg }); - r->setAttributes(cast(theFunc.getCallee())->getAttributes()); + r->setAttributes(theFunc->getAttributes()); ctx.builder.CreateRet(r); return f; } @@ -3933,7 +4429,7 @@ static void emit_cfunc_invalidate( jl_value_t *calltype, jl_value_t *rettype, size_t nargs, jl_codegen_params_t ¶ms, - Function *target=jlapplygeneric_func) + Function *target) { jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = gf_thunk; @@ -4022,6 +4518,16 @@ static void emit_cfunc_invalidate( } } +static void emit_cfunc_invalidate( + Function *gf_thunk, jl_returninfo_t::CallingConv cc, unsigned return_roots, + jl_value_t *calltype, jl_value_t *rettype, + size_t nargs, + jl_codegen_params_t ¶ms) +{ + emit_cfunc_invalidate(gf_thunk, cc, return_roots, calltype, rettype, nargs, params, + prepare_call_in(gf_thunk->getParent(), jlapplygeneric_func)); +} + static Function* gen_cfun_wrapper( Module *into, jl_codegen_params_t ¶ms, const function_sig_t &sig, jl_value_t *ff, const char *aliasname, @@ -4328,7 +4834,7 @@ static Function* gen_cfun_wrapper( ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_generic); } - Value *ret = emit_jlcall(ctx, prepare_call(jlapplygeneric_func).getCallee(), NULL, inputargs, nargs + 1, JLCALL_F_CC); + Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, JLCALL_F_CC); if (age_ok) { ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_after); @@ -4718,12 +5224,8 @@ void jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declr // restore a ccallable from the system image void *addr; int found = jl_dlsym(sysimg_handle, name, &addr, 0); - if (found) { - FunctionType *ftype = sig.functype(); - Function *F = Function::Create(ftype, GlobalVariable::ExternalLinkage, name); - add_named_global(F, addr); - delete F; - } + if (found) + add_named_global(name, addr); } else { jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0); @@ -5624,7 +6126,7 @@ static std::pair, jl_llvm_functions_t> emit_varinfo_assign(ctx, vi, tuple); } else { - restTuple = emit_jlcall(ctx, prepare_call(jltuple_func).getCallee(), maybe_decay_untracked(V_null), + restTuple = emit_jlcall(ctx, jltuple_func, maybe_decay_untracked(V_null), vargs, ctx.nvargs, JLCALL_F_CC); jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, vi.value.typ); emit_varinfo_assign(ctx, vi, tuple); @@ -5632,14 +6134,15 @@ static std::pair, jl_llvm_functions_t> } else { // restarg = jl_f_tuple(NULL, &args[nreq], nargs - nreq) + Function *F = prepare_call(jltuple_func); restTuple = - ctx.builder.CreateCall(prepare_call(jltuple_func), + ctx.builder.CreateCall(F, { maybe_decay_untracked(V_null), ctx.builder.CreateInBoundsGEP(T_prjlvalue, argArray, ConstantInt::get(T_size, nreq - 1)), ctx.builder.CreateSub(argCount, ConstantInt::get(T_int32, nreq - 1)) }); - restTuple->setAttributes(jltuple_func->getAttributes()); + restTuple->setAttributes(F->getAttributes()); ctx.builder.CreateStore(restTuple, vi.boxroot); } } @@ -6652,33 +7155,20 @@ std::pair tbaa_make_child(const char *name, MDNode *parent=null return std::make_pair(n, scalar); } -static GlobalVariable *global_to_llvm(const std::string &cname, void *addr, Module *m) -{ - GlobalVariable *gv = - new GlobalVariable(*m, T_pjlvalue, true, - GlobalVariable::ExternalLinkage, NULL, cname); - add_named_global(gv, addr); - return gv; -} -llvm::SmallVector, 16> gv_for_global; -static GlobalVariable *global_jlvalue_to_llvm(const std::string &cname, jl_value_t **addr, Module *m) +std::vector> gv_for_global; +static void global_jlvalue_to_llvm(JuliaVariable *var, jl_value_t **addr) { - GlobalVariable *gv = global_to_llvm(cname, (void*)addr, m); - gv_for_global.push_back(std::make_pair(addr, gv)); - return gv; + gv_for_global.push_back(std::make_pair(addr, var)); } -static GlobalVariable *julia_const_gv(jl_value_t *val) +static JuliaVariable *julia_const_gv(jl_value_t *val) { - for (auto& kv : gv_for_global) { + for (auto &kv : gv_for_global) { if (*kv.first == val) return kv.second; } return nullptr; } -// TODO: delete this -extern "C" void jl_fptr_to_llvm(void *fptr, jl_code_instance_t *lam, int specsig) { } - static void init_julia_llvm_meta(void) { tbaa_gcframe = tbaa_make_child("jtbaa_gcframe").first; @@ -6708,15 +7198,6 @@ static void init_julia_llvm_meta(void) Thunk = Attribute::get(jl_LLVMContext, "thunk"); } -static Function *jlcall_func_to_llvm(const std::string &cname, jl_fptr_args_t addr, Module *m) -{ - Function *f = Function::Create(jl_func_sig, Function::ExternalLinkage, cname, m); - add_return_attr(f, Attribute::NonNull); - f->addFnAttr(Thunk); - add_named_global(f, addr); - return f; -} - static void init_julia_llvm_env(Module *m) { // every variable or function mapped in this function must be @@ -6750,8 +7231,6 @@ static void init_julia_llvm_env(Module *m) T_void = Type::getVoidTy(jl_LLVMContext); T_pvoidfunc = FunctionType::get(T_void, /*isVarArg*/false)->getPointerTo(); - auto T_pint8_derived = PointerType::get(T_int8, AddressSpace::Derived); - // add needed base debugging definitions to our LLVM environment DIBuilder dbuilder(*m); DIFile *julia_h = dbuilder.createFile("julia.h", ""); @@ -6818,556 +7297,104 @@ static void init_julia_llvm_env(Module *m) jl_array_llvmt = StructType::create(jl_LLVMContext, makeArrayRef(vaelts), "jl_array_t"); jl_parray_llvmt = PointerType::get(jl_array_llvmt, 0); +} - global_to_llvm("__stack_chk_guard", (void*)&__stack_chk_guard, m); - Function *jl__stack_chk_fail = - Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, - "__stack_chk_fail", m); - jl__stack_chk_fail->setDoesNotReturn(); - add_named_global(jl__stack_chk_fail, &__stack_chk_fail); - - global_jlvalue_to_llvm("jl_true", &jl_true, m); - global_jlvalue_to_llvm("jl_false", &jl_false, m); - global_jlvalue_to_llvm("jl_emptysvec", (jl_value_t**)&jl_emptysvec, m); - global_jlvalue_to_llvm("jl_emptytuple", &jl_emptytuple, m); - global_jlvalue_to_llvm("jl_diverror_exception", &jl_diverror_exception, m); - global_jlvalue_to_llvm("jl_undefref_exception", &jl_undefref_exception, m); - - jlRTLD_DEFAULT_var = - new GlobalVariable(*m, T_pint8, - true, GlobalVariable::ExternalLinkage, - NULL, "jl_RTLD_DEFAULT_handle"); +static void init_jit_functions(void) +{ + add_named_global(jlstack_chk_guard_var, &__stack_chk_guard); add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle); #ifdef _OS_WINDOWS_ - jlexe_var = - new GlobalVariable(*m, T_pint8, - true, GlobalVariable::ExternalLinkage, - NULL, "jl_exe_handle"); add_named_global(jlexe_var, &jl_exe_handle); - jldll_var = - new GlobalVariable(*m, T_pint8, - true, GlobalVariable::ExternalLinkage, - NULL, "jl_dl_handle"); add_named_global(jldll_var, &jl_dl_handle); #endif - - jltls_states_func = Function::Create(FunctionType::get(PointerType::get(T_ppjlvalue, 0), false), - Function::ExternalLinkage, "julia.ptls_states"); - add_named_global(jltls_states_func, (void*)NULL, /*dllimport*/false); - - std::vector args1(0); - args1.push_back(T_pint8); - jlerror_func = - Function::Create(FunctionType::get(T_void, args1, false), - Function::ExternalLinkage, - "jl_error", m); - jlerror_func->setDoesNotReturn(); + global_jlvalue_to_llvm(new JuliaVariable{"jl_true", true, get_pjlvalue}, &jl_true); + global_jlvalue_to_llvm(new JuliaVariable{"jl_false", true, get_pjlvalue}, &jl_false); + global_jlvalue_to_llvm(new JuliaVariable{"jl_emptysvec", true, get_pjlvalue}, (jl_value_t**)&jl_emptysvec); + global_jlvalue_to_llvm(new JuliaVariable{"jl_emptytuple", true, get_pjlvalue}, &jl_emptytuple); + global_jlvalue_to_llvm(new JuliaVariable{"jl_diverror_exception", true, get_pjlvalue}, &jl_diverror_exception); + global_jlvalue_to_llvm(new JuliaVariable{"jl_undefref_exception", true, get_pjlvalue}, &jl_undefref_exception); + add_named_global(jlgetworld_global, &jl_world_counter); + add_named_global("__stack_chk_fail", &__stack_chk_fail); + add_named_global(jltls_states_func, (void*)NULL); add_named_global(jlerror_func, &jl_error); - - std::vector args1_(0); - args1_.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - jlthrow_func = - Function::Create(FunctionType::get(T_void, args1_, false), - Function::ExternalLinkage, - "jl_throw", m); - jlthrow_func->setDoesNotReturn(); add_named_global(jlthrow_func, &jl_throw); - - // Symbols are not gc-tracked, but we'll treat them as callee rooted anyway, - // because they may come from a gc-rooted location - jlundefvarerror_func = - Function::Create(FunctionType::get(T_void, args1_, false), - Function::ExternalLinkage, - "jl_undefined_var_error", m); - jlundefvarerror_func->setDoesNotReturn(); add_named_global(jlundefvarerror_func, &jl_undefined_var_error); - - std::vector args2_boundserrorv(0); - args2_boundserrorv.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - args2_boundserrorv.push_back(T_psize); - args2_boundserrorv.push_back(T_size); - jlboundserrorv_func = - Function::Create(FunctionType::get(T_void, args2_boundserrorv, false), - Function::ExternalLinkage, - "jl_bounds_error_ints", m); - jlboundserrorv_func->setDoesNotReturn(); add_named_global(jlboundserrorv_func, &jl_bounds_error_ints); - - std::vector args2_boundserror(0); - args2_boundserror.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - args2_boundserror.push_back(T_size); - jlboundserror_func = - Function::Create(FunctionType::get(T_void, args2_boundserror, false), - Function::ExternalLinkage, - "jl_bounds_error_int", m); - jlboundserror_func->setDoesNotReturn(); add_named_global(jlboundserror_func, &jl_bounds_error_int); - - std::vector args3_vboundserror(0); - args3_vboundserror.push_back(T_pprjlvalue); - args3_vboundserror.push_back(T_size); - args3_vboundserror.push_back(T_size); - jlvboundserror_func = - Function::Create(FunctionType::get(T_void, args3_vboundserror, false), - Function::ExternalLinkage, - "jl_bounds_error_tuple_int", m); - jlvboundserror_func->setDoesNotReturn(); add_named_global(jlvboundserror_func, &jl_bounds_error_tuple_int); - - std::vector args3_uboundserror(0); - args3_uboundserror.push_back(T_pint8_derived); - args3_uboundserror.push_back(T_pjlvalue); - args3_uboundserror.push_back(T_size); - jluboundserror_func = - Function::Create(FunctionType::get(T_void, args3_uboundserror, false), - Function::ExternalLinkage, - "jl_bounds_error_unboxed_int", m); - jluboundserror_func->setDoesNotReturn(); add_named_global(jluboundserror_func, &jl_bounds_error_unboxed_int); - - jlnew_func = - Function::Create(jl_func_sig, Function::ExternalLinkage, - "jl_new_structv", m); - add_return_attr(jlnew_func, Attribute::NonNull); - jlnew_func->addFnAttr(Thunk); add_named_global(jlnew_func, &jl_new_structv); - - std::vector args_2rptrs_(0); - args_2rptrs_.push_back(T_prjlvalue); - args_2rptrs_.push_back(T_prjlvalue); - jlsplatnew_func = - Function::Create(FunctionType::get(T_prjlvalue, args_2rptrs_, false), - Function::ExternalLinkage, - "jl_new_structt", m); - add_return_attr(jlsplatnew_func, Attribute::NonNull); - jlsplatnew_func->addFnAttr(Thunk); add_named_global(jlsplatnew_func, &jl_new_structt); - - std::vector args2(0); - args2.push_back(T_pint8); -#ifndef _OS_WINDOWS_ - args2.push_back(T_int32); -#endif - setjmp_func = - Function::Create(FunctionType::get(T_int32, args2, false), - Function::ExternalLinkage, jl_setjmp_name, m); - setjmp_func->addFnAttr(Attribute::ReturnsTwice); add_named_global(setjmp_func, &jl_setjmp_f); - - std::vector args_memcmp(0); - args_memcmp.push_back(T_pint8); - args_memcmp.push_back(T_pint8); - args_memcmp.push_back(T_size); - memcmp_func = - Function::Create(FunctionType::get(T_int32, args_memcmp, false), - Function::ExternalLinkage, "memcmp", m); - memcmp_func->addFnAttr(Attribute::ReadOnly); - memcmp_func->addFnAttr(Attribute::NoUnwind); - memcmp_func->addFnAttr(Attribute::ArgMemOnly); add_named_global(memcmp_func, &memcmp); - // TODO: inferLibFuncAttributes(*memcmp_func, TLI); - - std::vector te_args(0); - te_args.push_back(T_pint8); - te_args.push_back(T_prjlvalue); - te_args.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - jltypeerror_func = - Function::Create(FunctionType::get(T_void, te_args, false), - Function::ExternalLinkage, - "jl_type_error", m); - jltypeerror_func->setDoesNotReturn(); add_named_global(jltypeerror_func, &jl_type_error); - - std::vector args_2ptrs(0); - args_2ptrs.push_back(T_pjlvalue); - args_2ptrs.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - jlcheckassign_func = - Function::Create(FunctionType::get(T_void, args_2ptrs, false), - Function::ExternalLinkage, - "jl_checked_assignment", m); add_named_global(jlcheckassign_func, &jl_checked_assignment); - - std::vector args_1binding(0); - args_1binding.push_back(T_pjlvalue); - jldeclareconst_func = - Function::Create(FunctionType::get(T_void, args_1binding, false), - Function::ExternalLinkage, - "jl_declare_constant", m); add_named_global(jldeclareconst_func, &jl_declare_constant); - - std::vector args_2ptrs_(0); - args_2ptrs_.push_back(T_pjlvalue); - args_2ptrs_.push_back(T_pjlvalue); - jlgetbindingorerror_func = - Function::Create(FunctionType::get(T_pjlvalue, args_2ptrs_, false), - Function::ExternalLinkage, - "jl_get_binding_or_error", m); add_named_global(jlgetbindingorerror_func, &jl_get_binding_or_error); - - jlboundp_func = - Function::Create(FunctionType::get(T_int32, args_2ptrs_, false), - Function::ExternalLinkage, - "jl_boundp", m); add_named_global(jlboundp_func, &jl_boundp); - - builtin_func_map[jl_f_is] = jlcall_func_to_llvm("jl_f_is", &jl_f_is, m); - builtin_func_map[jl_f_typeof] = jlcall_func_to_llvm("jl_f_typeof", &jl_f_typeof, m); - builtin_func_map[jl_f_sizeof] = jlcall_func_to_llvm("jl_f_sizeof", &jl_f_sizeof, m); - builtin_func_map[jl_f_issubtype] = jlcall_func_to_llvm("jl_f_issubtype", &jl_f_issubtype, m); - builtin_func_map[jl_f_isa] = jlcall_func_to_llvm("jl_f_isa", &jl_f_isa, m); - builtin_func_map[jl_f_typeassert] = jlcall_func_to_llvm("jl_f_typeassert", &jl_f_typeassert, m); - builtin_func_map[jl_f_ifelse] = jlcall_func_to_llvm("jl_f_ifelse", &jl_f_ifelse, m); - builtin_func_map[jl_f__apply] = jlcall_func_to_llvm("jl_f__apply", &jl_f__apply, m); - builtin_func_map[jl_f__apply_iterate] = jlcall_func_to_llvm("jl_f__apply_iterate", &jl_f__apply_iterate, m); - builtin_func_map[jl_f__apply_pure] = jlcall_func_to_llvm("jl_f__apply_pure", &jl_f__apply_pure, m); - builtin_func_map[jl_f__apply_latest] = jlcall_func_to_llvm("jl_f__apply_latest", &jl_f__apply_latest, m); - builtin_func_map[jl_f_throw] = jlcall_func_to_llvm("jl_f_throw", &jl_f_throw, m); - builtin_func_map[jl_f_tuple] = jlcall_func_to_llvm("jl_f_tuple", &jl_f_tuple, m); - builtin_func_map[jl_f_svec] = jlcall_func_to_llvm("jl_f_svec", &jl_f_svec, m); - builtin_func_map[jl_f_applicable] = jlcall_func_to_llvm("jl_f_applicable", &jl_f_applicable, m); - builtin_func_map[jl_f_invoke] = jlcall_func_to_llvm("jl_f_invoke", &jl_f_invoke, m); - builtin_func_map[jl_f_invoke_kwsorter] = jlcall_func_to_llvm("jl_f_invoke_kwsorter", &jl_f_invoke_kwsorter, m); - builtin_func_map[jl_f_isdefined] = jlcall_func_to_llvm("jl_f_isdefined", &jl_f_isdefined, m); - builtin_func_map[jl_f_getfield] = jlcall_func_to_llvm("jl_f_getfield", &jl_f_getfield, m); - builtin_func_map[jl_f_setfield] = jlcall_func_to_llvm("jl_f_setfield", &jl_f_setfield, m); - builtin_func_map[jl_f_fieldtype] = jlcall_func_to_llvm("jl_f_fieldtype", &jl_f_fieldtype, m); - builtin_func_map[jl_f_nfields] = jlcall_func_to_llvm("jl_f_nfields", &jl_f_nfields, m); - builtin_func_map[jl_f__expr] = jlcall_func_to_llvm("jl_f__expr", &jl_f__expr, m); - builtin_func_map[jl_f__typevar] = jlcall_func_to_llvm("jl_f__typevar", &jl_f__typevar, m); - builtin_func_map[jl_f_arrayref] = jlcall_func_to_llvm("jl_f_arrayref", &jl_f_arrayref, m); - builtin_func_map[jl_f_const_arrayref] = jlcall_func_to_llvm("jl_f_const_arrayref", &jl_f_arrayref, m); - builtin_func_map[jl_f_arrayset] = jlcall_func_to_llvm("jl_f_arrayset", &jl_f_arrayset, m); - builtin_func_map[jl_f_arraysize] = jlcall_func_to_llvm("jl_f_arraysize", &jl_f_arraysize, m); - builtin_func_map[jl_f_apply_type] = jlcall_func_to_llvm("jl_f_apply_type", &jl_f_apply_type, m); - jltuple_func = builtin_func_map[jl_f_tuple]; - jlgetfield_func = builtin_func_map[jl_f_getfield]; - - jlapplygeneric_func = Function::Create(jl_func_sig, - Function::ExternalLinkage, - "jl_apply_generic", m); - add_return_attr(jlapplygeneric_func, Attribute::NonNull); - jlapplygeneric_func->addFnAttr(Thunk); + for (auto it : builtin_func_map) + add_named_global(it.second, it.first); add_named_global(jlapplygeneric_func, &jl_apply_generic); - - std::vector invokeargs(0); - invokeargs.push_back(T_prjlvalue); - invokeargs.push_back(T_pprjlvalue); - invokeargs.push_back(T_uint32); - invokeargs.push_back(T_prjlvalue); - jlinvoke_func = Function::Create(FunctionType::get(T_prjlvalue, invokeargs, false), - Function::ExternalLinkage, - "jl_invoke", m); - add_return_attr(jlinvoke_func, Attribute::NonNull); - jlinvoke_func->addAttribute(2, Attribute::ReadOnly); - jlinvoke_func->addAttribute(2, Attribute::NoCapture); add_named_global(jlinvoke_func, &jl_invoke); - - std::vector exp_args(0); - exp_args.push_back(T_int1); - expect_func = Intrinsic::getDeclaration(m, Intrinsic::expect, exp_args); - - std::vector args_topeval(0); - args_topeval.push_back(T_pjlvalue); - args_topeval.push_back(T_pjlvalue); - jltopeval_func = - Function::Create(FunctionType::get(T_pjlvalue, args_topeval, false), - Function::ExternalLinkage, - "jl_toplevel_eval", m); - add_return_attr(jltopeval_func, Attribute::NonNull); add_named_global(jltopeval_func, &jl_toplevel_eval); - - std::vector args_copyast(0); - args_copyast.push_back(T_prjlvalue); - jlcopyast_func = - Function::Create(FunctionType::get(T_prjlvalue, args_copyast, false), - Function::ExternalLinkage, - "jl_copy_ast", m); - add_return_attr(jlcopyast_func, Attribute::NonNull); add_named_global(jlcopyast_func, &jl_copy_ast); - - std::vector args5(0); - args5.push_back(T_size); - jlnsvec_func = - Function::Create(FunctionType::get(T_pjlvalue, args5, true), - Function::ExternalLinkage, - "jl_svec", m); - add_return_attr(jlnsvec_func, Attribute::NonNull); - add_named_global(jlnsvec_func, &jl_svec); - - std::vector mdargs(0); - mdargs.push_back(T_prjlvalue); - mdargs.push_back(T_prjlvalue); - mdargs.push_back(T_pjlvalue); - jlmethod_func = - Function::Create(FunctionType::get(T_void, mdargs, false), - Function::ExternalLinkage, - "jl_method_def", m); + //add_named_global(jlnsvec_func, &jl_svec); add_named_global(jlmethod_func, &jl_method_def); - - std::vector funcdefargs(0); - funcdefargs.push_back(T_pjlvalue); - funcdefargs.push_back(T_pjlvalue); - funcdefargs.push_back(T_pprjlvalue); - funcdefargs.push_back(T_pjlvalue); - funcdefargs.push_back(T_pjlvalue); - jlgenericfunction_func = - Function::Create(FunctionType::get(T_prjlvalue, funcdefargs, false), - Function::ExternalLinkage, - "jl_generic_function_def", m); add_named_global(jlgenericfunction_func, &jl_generic_function_def); - - std::vector ehargs(0); - ehargs.push_back(T_pint8); - jlenter_func = - Function::Create(FunctionType::get(T_void, ehargs, false), - Function::ExternalLinkage, - "jl_enter_handler", m); add_named_global(jlenter_func, &jl_enter_handler); - - jl_current_exception_func = - Function::Create(FunctionType::get(T_prjlvalue, false), - Function::ExternalLinkage, - "jl_current_exception", m); add_named_global(jl_current_exception_func, &jl_current_exception); - -#ifdef _OS_WINDOWS_ -#ifndef FORCE_ELF -#if defined(_CPU_X86_64_) -#if defined(_COMPILER_GCC_) - Function *chkstk_func = Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, "___chkstk_ms", m); - add_named_global(chkstk_func, &___chkstk_ms, /*dllimport*/false); -#else - Function *chkstk_func = Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, "__chkstk", m); - add_named_global(chkstk_func, &__chkstk, /*dllimport*/false); -#endif -#else -#if defined(_COMPILER_GCC_) - Function *chkstk_func = Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, "_alloca", m); - add_named_global(chkstk_func, &_alloca, /*dllimport*/false); -#else - Function *chkstk_func = Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, "_chkstk", m); - add_named_global(chkstk_func, &_chkstk, /*dllimport*/false); -#endif -#endif -#endif -#endif - - std::vector lhargs(0); - lhargs.push_back(T_int32); - jlleave_func = - Function::Create(FunctionType::get(T_void, lhargs, false), - Function::ExternalLinkage, - "jl_pop_handler", m); add_named_global(jlleave_func, &jl_pop_handler); - - jl_restore_excstack_func = - Function::Create(FunctionType::get(T_void, T_size, false), - Function::ExternalLinkage, - "jl_restore_excstack", m); add_named_global(jl_restore_excstack_func, &jl_restore_excstack); - - jl_excstack_state_func = - Function::Create(FunctionType::get(T_size, false), - Function::ExternalLinkage, - "jl_excstack_state", m); add_named_global(jl_excstack_state_func, &jl_excstack_state); - - std::vector args_2vals_callee_rooted(0); - args_2vals_callee_rooted.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - args_2vals_callee_rooted.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); - jlegal_func = - Function::Create(FunctionType::get(T_int32, args_2vals_callee_rooted, false), - Function::ExternalLinkage, - "jl_egal", m); add_named_global(jlegal_func, &jl_egal); - - std::vector args_2vals_tracked(0); - args_2vals_tracked.push_back(T_prjlvalue); - args_2vals_tracked.push_back(T_prjlvalue); - jlisa_func = - Function::Create(FunctionType::get(T_int32, args_2vals_tracked, false), - Function::ExternalLinkage, - "jl_isa", m); add_named_global(jlisa_func, &jl_isa); - - jlsubtype_func = - Function::Create(FunctionType::get(T_int32, args_2vals_tracked, false), - Function::ExternalLinkage, - "jl_subtype", m); add_named_global(jlsubtype_func, &jl_subtype); - - jltypeassert_func = Function::Create(FunctionType::get(T_void, args_2vals_tracked, false), - Function::ExternalLinkage, - "jl_typeassert", m); add_named_global(jltypeassert_func, &jl_typeassert); - - std::vector applytype_args(0); - applytype_args.push_back(T_pjlvalue); - applytype_args.push_back(T_pjlvalue); - applytype_args.push_back(T_pprjlvalue); - jlapplytype_func = - Function::Create(FunctionType::get(T_prjlvalue, applytype_args, false), - Function::ExternalLinkage, - "jl_instantiate_type_in_env", m); - add_return_attr(jlapplytype_func, Attribute::NonNull); add_named_global(jlapplytype_func, &jl_instantiate_type_in_env); - - std::vector objectid__args(0); - objectid__args.push_back(T_prjlvalue); - objectid__args.push_back(T_pint8_derived); - jl_object_id__func = - Function::Create(FunctionType::get(T_size, objectid__args, false), - Function::ExternalLinkage, - "jl_object_id_", m); add_named_global(jl_object_id__func, &jl_object_id_); - - std::vector gc_alloc_args(0); - gc_alloc_args.push_back(T_pint8); - gc_alloc_args.push_back(T_size); - gc_alloc_args.push_back(T_prjlvalue); - jl_alloc_obj_func = Function::Create(FunctionType::get(T_prjlvalue, gc_alloc_args, false), - Function::ExternalLinkage, - "julia.gc_alloc_obj"); - add_return_attr(jl_alloc_obj_func, Attribute::NoAlias); - add_return_attr(jl_alloc_obj_func, Attribute::NonNull); - jl_alloc_obj_func->addFnAttr(Attribute::getWithAllocSizeArgs(jl_LLVMContext, 1, None)); // returns %1 bytes - add_named_global(jl_alloc_obj_func, (void*)NULL, /*dllimport*/false); - - std::vector newbits_args(0); - newbits_args.push_back(T_prjlvalue); - newbits_args.push_back(T_pint8); - jl_newbits_func = Function::Create(FunctionType::get(T_prjlvalue, newbits_args, false), - Function::ExternalLinkage, - "jl_new_bits"); - add_return_attr(jl_newbits_func, Attribute::NoAlias); - add_return_attr(jl_newbits_func, Attribute::NonNull); + add_named_global(jl_alloc_obj_func, (void*)NULL); add_named_global(jl_newbits_func, (void*)jl_new_bits); - - jl_loopinfo_marker_func = Function::Create(FunctionType::get(T_void, {}, false), - Function::ExternalLinkage, - "julia.loopinfo_marker"); - jl_loopinfo_marker_func->addFnAttr(Attribute::NoUnwind); - jl_loopinfo_marker_func->addFnAttr(Attribute::NoRecurse); - jl_loopinfo_marker_func->addFnAttr(Attribute::InaccessibleMemOnly); - - jl_typeof_func = Function::Create(FunctionType::get(T_prjlvalue, {T_prjlvalue}, false), - Function::ExternalLinkage, - "julia.typeof"); - jl_typeof_func->addFnAttr(Attribute::ReadOnly); - jl_typeof_func->addFnAttr(Attribute::NoUnwind); - jl_typeof_func->addFnAttr(Attribute::ArgMemOnly); - jl_typeof_func->addFnAttr(Attribute::NoRecurse); - add_return_attr(jl_typeof_func, Attribute::NonNull); - add_named_global(jl_typeof_func, (void*)NULL, /*dllimport*/false); - - jl_write_barrier_func = Function::Create(FunctionType::get(T_void, {T_prjlvalue,}, true), - Function::ExternalLinkage, - "julia.write_barrier"); - jl_write_barrier_func->addFnAttr(Attribute::InaccessibleMemOnly); - jl_write_barrier_func->addFnAttr(Attribute::NoUnwind); - jl_write_barrier_func->addFnAttr(Attribute::NoRecurse); - add_named_global(jl_write_barrier_func, (void*)NULL, /*dllimport*/false); - - std::vector dlsym_args(0); - dlsym_args.push_back(T_pint8); - dlsym_args.push_back(T_pint8); - dlsym_args.push_back(PointerType::get(T_pint8,0)); - jldlsym_func = - Function::Create(FunctionType::get(T_pvoidfunc, dlsym_args, false), - Function::ExternalLinkage, - "jl_load_and_lookup", m); + add_named_global(jl_loopinfo_marker_func, (void*)NULL); + add_named_global(jl_typeof_func, (void*)NULL); + add_named_global(jl_write_barrier_func, (void*)NULL); add_named_global(jldlsym_func, &jl_load_and_lookup); - - std::vector getcfunctiontrampoline_args(0); - getcfunctiontrampoline_args.push_back(T_prjlvalue); // f (object) - getcfunctiontrampoline_args.push_back(T_pjlvalue); // result - getcfunctiontrampoline_args.push_back(T_pint8); // cache - getcfunctiontrampoline_args.push_back(T_pjlvalue); // fill - getcfunctiontrampoline_args.push_back(FunctionType::get(T_pint8, { T_pint8, T_ppjlvalue }, false)->getPointerTo()); // trampoline - getcfunctiontrampoline_args.push_back(T_pjlvalue); // env - getcfunctiontrampoline_args.push_back(T_pprjlvalue); // vals - jlgetcfunctiontrampoline_func = - Function::Create(FunctionType::get(T_prjlvalue, getcfunctiontrampoline_args, false), - Function::ExternalLinkage, - "jl_get_cfunction_trampoline", m); - add_return_attr(jlgetcfunctiontrampoline_func, Attribute::NonNull); add_named_global(jlgetcfunctiontrampoline_func, &jl_get_cfunction_trampoline); - - std::vector getnthfld_args(0); - getnthfld_args.push_back(T_prjlvalue); - getnthfld_args.push_back(T_size); - jlgetnthfieldchecked_func = - Function::Create(FunctionType::get(T_prjlvalue, getnthfld_args, false), - Function::ExternalLinkage, - "jl_get_nth_field_checked", m); - add_return_attr(jlgetnthfieldchecked_func, Attribute::NonNull); add_named_global(jlgetnthfieldchecked_func, &jl_get_nth_field_checked); - - diff_gc_total_bytes_func = - Function::Create(FunctionType::get(T_int64, false), - Function::ExternalLinkage, - "jl_gc_diff_total_bytes", m); add_named_global(diff_gc_total_bytes_func, &jl_gc_diff_total_bytes); - - sync_gc_total_bytes_func = - Function::Create(FunctionType::get(T_int64, {T_int64}, false), - Function::ExternalLinkage, - "jl_gc_sync_total_bytes", m); add_named_global(sync_gc_total_bytes_func, &jl_gc_sync_total_bytes); - - - std::vector array_owner_args(0); - array_owner_args.push_back(T_prjlvalue); - jlarray_data_owner_func = - Function::Create(FunctionType::get(T_prjlvalue, array_owner_args, false), - Function::ExternalLinkage, - "jl_array_data_owner", m); - jlarray_data_owner_func->addFnAttr(Attribute::ReadOnly); - jlarray_data_owner_func->addFnAttr(Attribute::NoUnwind); - add_return_attr(jlarray_data_owner_func, Attribute::NonNull); add_named_global(jlarray_data_owner_func, &jl_array_data_owner); + add_named_global(gcroot_flush_func, (void*)NULL); + add_named_global(gc_preserve_begin_func, (void*)NULL); + add_named_global(gc_preserve_end_func, (void*)NULL); + add_named_global(pointer_from_objref_func, (void*)NULL); + add_named_global(except_enter_func, (void*)NULL); - gcroot_flush_func = Function::Create(FunctionType::get(T_void, false), - Function::ExternalLinkage, - "julia.gcroot_flush"); - add_named_global(gcroot_flush_func, (void*)NULL, /*dllimport*/false); - - gc_preserve_begin_func = Function::Create(FunctionType::get(Type::getTokenTy(jl_LLVMContext), - ArrayRef(), true), - Function::ExternalLinkage, - "llvm.julia.gc_preserve_begin"); - add_named_global(gc_preserve_begin_func, (void*)NULL, /*dllimport*/false); - - gc_preserve_end_func = Function::Create(FunctionType::get(T_void, - ArrayRef(Type::getTokenTy(jl_LLVMContext)), false), - Function::ExternalLinkage, - "llvm.julia.gc_preserve_end"); - add_named_global(gc_preserve_end_func, (void*)NULL, /*dllimport*/false); - - pointer_from_objref_func = Function::Create(FunctionType::get(T_pjlvalue, - ArrayRef(PointerType::get(T_jlvalue, AddressSpace::Derived)), false), - Function::ExternalLinkage, - "julia.pointer_from_objref"); - pointer_from_objref_func->addFnAttr(Attribute::ReadNone); - pointer_from_objref_func->addFnAttr(Attribute::NoUnwind); - add_named_global(pointer_from_objref_func, (void*)NULL, /*dllimport*/false); - - except_enter_func = Function::Create(FunctionType::get(T_int32, false), - Function::ExternalLinkage, - "julia.except_enter"); - except_enter_func->addFnAttr(Attribute::ReturnsTwice); - add_named_global(except_enter_func, (void*)NULL, /*dllimport*/false); - - jlgetworld_global = - new GlobalVariable(*m, T_size, - false, GlobalVariable::ExternalLinkage, - NULL, "jl_world_counter"); - add_named_global(jlgetworld_global, &jl_world_counter); +#ifdef _OS_WINDOWS_ +#ifndef FORCE_ELF +#if defined(_CPU_X86_64_) +#if defined(_COMPILER_GCC_) + add_named_global("___chkstk_ms", &___chkstk_ms); +#else + add_named_global("__chkstk", &__chkstk); +#endif +#else +#if defined(_COMPILER_GCC_) + add_named_global("_alloca", &_alloca); +#else + add_named_global("_chkstk", &_chkstk); +#endif +#endif +#endif +#endif + +#define BOX_F(ct) add_named_global("jl_box_"#ct, &jl_box_##ct); + BOX_F(int8); BOX_F(uint8); + BOX_F(int16); BOX_F(uint16); + BOX_F(int32); BOX_F(uint32); + BOX_F(int64); BOX_F(uint64); + BOX_F(float32); BOX_F(float64); + BOX_F(char); BOX_F(ssavalue); +#undef BOX_F } extern "C" void jl_init_llvm(void) @@ -7527,19 +7554,13 @@ extern "C" void jl_init_codegen(void) jl_init_llvm(); // Now that the execution engine exists, initialize all modules jl_init_jit(); + init_jit_functions(); + Module *m = new Module("julia", jl_LLVMContext); jl_setup_module(m); init_julia_llvm_env(m); - SBOX_F_PERM(int8,int8); UBOX_F_PERM(uint8,uint8); - SBOX_F(int16,int16); UBOX_F(uint16,uint16); - SBOX_F(int32,int32); UBOX_F(uint32,uint32); - SBOX_F(int64,int64); UBOX_F(uint64,uint64); - BOX_F(float32,float32,T_prjlvalue); BOX_F(float64,float64,T_prjlvalue); - UBOX_F(char,char); - UBOX_F(ssavalue,size); - - jl_init_intrinsic_functions_codegen(m); + jl_init_intrinsic_functions_codegen(); } extern "C" void jl_teardown_codegen() diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 9cff44db3190b..28e06d72ee1d1 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -7,32 +7,28 @@ namespace JL_I { #include "ccall.cpp" using namespace JL_I; -static Function *runtime_func[num_intrinsics]; + +FunctionType *get_intr_args1(LLVMContext &C) { return FunctionType::get(T_prjlvalue, {T_prjlvalue}, false); } +FunctionType *get_intr_args2(LLVMContext &C) { return FunctionType::get(T_prjlvalue, {T_prjlvalue, T_prjlvalue}, false); } +FunctionType *get_intr_args3(LLVMContext &C) { return FunctionType::get(T_prjlvalue, {T_prjlvalue, T_prjlvalue, T_prjlvalue}, false); } +FunctionType *get_intr_args4(LLVMContext &C) { return FunctionType::get(T_prjlvalue, {T_prjlvalue, T_prjlvalue, T_prjlvalue, T_prjlvalue}, false); } + +static JuliaFunction *runtime_func[num_intrinsics] = { +#define ADD_I(name, nargs) new JuliaFunction{"jl_"#name, get_intr_args##nargs}, +#define ADD_HIDDEN ADD_I +#define ALIAS(alias, base) nullptr, + INTRINSICS +#undef ADD_I +#undef ADD_HIDDEN +#undef ALIAS +}; + static bool float_func[num_intrinsics]; -static void jl_init_intrinsic_functions_codegen(Module *m) + +static void jl_init_intrinsic_functions_codegen(void) { - std::vector args1(0); \ - args1.push_back(T_prjlvalue); \ - std::vector args2(0); \ - args2.push_back(T_prjlvalue); \ - args2.push_back(T_prjlvalue); \ - std::vector args3(0); \ - args3.push_back(T_prjlvalue); \ - args3.push_back(T_prjlvalue); \ - args3.push_back(T_prjlvalue); \ - std::vector args4(0); \ - args4.push_back(T_prjlvalue); \ - args4.push_back(T_prjlvalue); \ - args4.push_back(T_prjlvalue); \ - args4.push_back(T_prjlvalue); - -#define ADD_I(name, nargs) do { \ - Function *func = Function::Create(FunctionType::get(T_prjlvalue, args##nargs, false), \ - Function::ExternalLinkage, "jl_"#name, m); \ - runtime_func[name] = func; \ - add_named_global(func, &jl_##name); \ - } while (0); -#define ADD_HIDDEN ADD_I +#define ADD_I(name, nargs) +#define ADD_HIDDEN(name, nargs) #define ALIAS(alias, base) runtime_func[alias] = runtime_func[base]; INTRINSICS #undef ADD_I @@ -414,7 +410,7 @@ static jl_value_t *staticeval_bitstype(const jl_cgval_t &targ) static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, const jl_cgval_t *argv, size_t nargs) { - FunctionCallee func = prepare_call(runtime_func[f]); + Function *func = prepare_call(runtime_func[f]); Value **argvalues = (Value**)alloca(sizeof(Value*) * nargs); for (size_t i = 0; i < nargs; ++i) { argvalues[i] = boxed(ctx, argv[i]); @@ -1284,27 +1280,3 @@ static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, Value **arg } assert(0 && "unreachable"); } - -#define BOX_F(ct,jl_ct,rt) \ - box_##ct##_func = boxfunc_llvm(ft1arg(rt, T_##jl_ct), \ - "jl_box_"#ct, &jl_box_##ct, m); - -#define SBOX_F(ct,jl_ct) BOX_F(ct,jl_ct,T_prjlvalue); box_##ct##_func->addAttribute(1, Attribute::SExt); -#define UBOX_F(ct,jl_ct) BOX_F(ct,jl_ct,T_prjlvalue); box_##ct##_func->addAttribute(1, Attribute::ZExt); -#define SBOX_F_PERM(ct,jl_ct) BOX_F(ct,jl_ct,T_pjlvalue); box_##ct##_func->addAttribute(1, Attribute::SExt); -#define UBOX_F_PERM(ct,jl_ct) BOX_F(ct,jl_ct,T_pjlvalue); box_##ct##_func->addAttribute(1, Attribute::ZExt); - -template -static Function *boxfunc_llvm(FunctionType *ft, const std::string &cname, - T *addr, Module *m) -{ - Function *f = - Function::Create(ft, Function::ExternalLinkage, cname, m); - add_named_global(f, addr); - return f; -} - -static FunctionType *ft1arg(Type *ret, Type *arg) -{ - return FunctionType::get(ret, { arg }, false); -} diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index af8364af25844..b7c4a78b2ddca 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -618,16 +618,12 @@ JuliaOJIT::JuliaOJIT(TargetMachine &TM) void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) { - bool successful = GlobalSymbolTable.insert(std::make_pair(Name, (void*)Addr)).second; + std::string MangleName = getMangledName(Name); + bool successful = GlobalSymbolTable.insert(std::make_pair(MangleName, (void*)Addr)).second; (void)successful; assert(successful); } -void JuliaOJIT::addGlobalMapping(const GlobalValue *GV, void *Addr) -{ - addGlobalMapping(getMangledName(GV), (uintptr_t)Addr); -} - void *JuliaOJIT::getPointerToGlobalIfAvailable(StringRef S) { SymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); @@ -991,16 +987,7 @@ static uint64_t getAddressForFunction(StringRef fname) } // helper function for adding a DLLImport (dlsym) address to the execution engine -void add_named_global(GlobalObject *gv, void *addr, bool dllimport) +void add_named_global(StringRef name, void *addr) { -#ifdef _OS_WINDOWS_ - // setting JL_DLLEXPORT correctly only matters when building a binary - if (dllimport && imaging_mode) { - assert(gv->getLinkage() == GlobalValue::ExternalLinkage); - // add the __declspec(dllimport) attribute - gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); - } -#endif // _OS_WINDOWS_ - - jl_ExecutionEngine->addGlobalMapping(gv, addr); + jl_ExecutionEngine->addGlobalMapping(name, (uint64_t)(uintptr_t)addr); } diff --git a/src/jitlayers.h b/src/jitlayers.h index ed957ad44b588..91ced69b210a2 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -28,6 +28,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool lowe void jl_finalize_module(std::unique_ptr m); void jl_merge_module(Module *dest, std::unique_ptr src); Module *jl_create_llvm_module(StringRef name); +GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M); typedef struct _jl_llvm_functions_t { std::string functionObject; // jlcall llvm Function name @@ -113,33 +114,7 @@ void jl_compile_workqueue( Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt, jl_codegen_params_t ¶ms); -static inline GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G) -{ - if (G->getParent() == M) - return G; - GlobalValue *local = M->getNamedValue(G->getName()); - if (!local) { - // Copy the GlobalVariable, but without the initializer, so it becomes a declaration - GlobalVariable *proto = new GlobalVariable(*M, G->getType()->getElementType(), - G->isConstant(), GlobalVariable::ExternalLinkage, - nullptr, G->getName(), nullptr, G->getThreadLocalMode()); - proto->copyAttributesFrom(G); - // DLLImport only needs to be set for the shadow module - // it just gets annoying in the JIT - proto->setDLLStorageClass(GlobalValue::DefaultStorageClass); - return proto; - } - return cast(local); -} - -void add_named_global(GlobalObject *gv, void *addr, bool dllimport); -template -static inline void add_named_global(GlobalObject *gv, T *addr, bool dllimport = true) -{ - // cast through integer to avoid c++ pedantic warning about casting between - // data and code pointers - add_named_global(gv, (void*)(uintptr_t)addr, dllimport); -} +void add_named_global(StringRef name, void *addr); static inline Constant *literal_static_pointer_val(const void *p, Type *T) { @@ -212,7 +187,6 @@ class JuliaOJIT { const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &LoadedObjectInfo); void addGlobalMapping(StringRef Name, uint64_t Addr); - void addGlobalMapping(const GlobalValue *GV, void *Addr); void *getPointerToGlobalIfAvailable(StringRef S); void *getPointerToGlobalIfAvailable(const GlobalValue *GV); void addModule(std::unique_ptr M); diff --git a/src/julia_internal.h b/src/julia_internal.h index 61123c4b4423d..7c2fe499de5a8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -636,7 +636,6 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller); uint32_t jl_module_next_counter(jl_module_t *m); -void jl_fptr_to_llvm(void *fptr, jl_code_instance_t *codeinst, int spec_abi); jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs); int jl_has_meta(jl_array_t *body, jl_sym_t *sym); diff --git a/src/staticdata.c b/src/staticdata.c index 6f436fcdc339c..4211df93d4f7f 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1189,7 +1189,6 @@ static void jl_update_all_fptrs(jl_serializer_state *s) else { codeinst->invoke = (jl_callptr_t)fptr; } - jl_fptr_to_llvm(fptr, codeinst, specfunc); } } jl_register_fptrs(sysimage_base, &fvars, linfos, sysimg_fvars_max);