From 16f53fef940a0265cc64cda70a4703281bdb5e52 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 24 Oct 2017 15:41:57 -0400 Subject: [PATCH] codegen: restructure, remove recursion also provides support for using a different code_native format, as a fallback, later we'll want to make this more configurable there are now several primary interfaces to native code: - codegen: mostly internal, support for translating IR to LLVM - jitlayers: manages compilation passes, especially at runtime - aotcompile: support for managing external code - disasm: pretty-printer for code objects - debuginfo: tracking for unwind info --- base/options.jl | 1 - base/reflection.jl | 10 +- src/Makefile | 21 +- src/anticodegen.c | 4 +- src/aotcompile.cpp | 441 +++++++ src/ccall.cpp | 83 +- src/cgutils.cpp | 55 +- src/codegen.cpp | 1435 ++++++++-------------- src/debuginfo.cpp | 26 - src/disasm.cpp | 98 +- src/dump.c | 9 +- src/gf.c | 113 +- src/jitlayers.cpp | 1034 +++++++++------- src/jitlayers.h | 84 +- src/jloptions.c | 6 - src/jltypes.c | 14 +- src/julia.h | 26 +- src/julia_internal.h | 90 +- src/method.c | 4 +- src/precompile.c | 67 +- src/staticdata.c | 77 +- src/threading.c | 12 +- stdlib/InteractiveUtils/src/codeview.jl | 46 +- stdlib/InteractiveUtils/test/runtests.jl | 24 +- test/core.jl | 4 +- test/reflection.jl | 16 - test/staged.jl | 4 +- 27 files changed, 2024 insertions(+), 1780 deletions(-) create mode 100644 src/aotcompile.cpp diff --git a/base/options.jl b/base/options.jl index 934921c9d63d4..bfa84194d0bd8 100644 --- a/base/options.jl +++ b/base/options.jl @@ -34,7 +34,6 @@ struct JLOptions bindto::Ptr{UInt8} outputbc::Ptr{UInt8} outputunoptbc::Ptr{UInt8} - outputjitbc::Ptr{UInt8} outputo::Ptr{UInt8} outputji::Ptr{UInt8} incremental::Int8 diff --git a/base/reflection.jl b/base/reflection.jl index f3d71ae09970f..92f1b895ad2c3 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -785,8 +785,6 @@ end # this type mirrors jl_cgparams_t (documented in julia.h) struct CodegenParams - cached::Cint - track_allocations::Cint code_coverage::Cint static_alloc::Cint @@ -796,14 +794,14 @@ struct CodegenParams module_activation::Any raise_exception::Any - CodegenParams(;cached::Bool=true, - track_allocations::Bool=true, code_coverage::Bool=true, + function CodegenParams(; track_allocations::Bool=true, code_coverage::Bool=true, static_alloc::Bool=true, prefer_specsig::Bool=false, - module_setup=nothing, module_activation=nothing, raise_exception=nothing) = - new(Cint(cached), + module_setup=nothing, module_activation=nothing, raise_exception=nothing) + return new( Cint(track_allocations), Cint(code_coverage), Cint(static_alloc), Cint(prefer_specsig), module_setup, module_activation, raise_exception) + end end # give a decent error message if we try to instantiate a staged function on non-leaf types diff --git a/src/Makefile b/src/Makefile index 90f7c0b725ca2..acd5ca90a2312 100644 --- a/src/Makefile +++ b/src/Makefile @@ -50,7 +50,7 @@ endif LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) -SRCS += codegen jitlayers disasm debuginfo llvm-simdloop llvm-ptls llvm-muladd \ +SRCS += codegen jitlayers aotcompile disasm debuginfo llvm-simdloop llvm-ptls llvm-muladd \ llvm-late-gc-lowering llvm-lower-handlers llvm-gc-invariant-verifier \ llvm-propagate-addrspaces llvm-multiversioning llvm-alloc-opt cgmemmgr \ llvm-api @@ -183,16 +183,23 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) # additional dependency links -$(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\ intrinsics.cpp jitlayers.h intrinsics.h debuginfo.h codegen_shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h) +$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-gc-invariant-verifier.o $(BUILDDIR)/llvm-gc-invariant-verifier.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-late-gc-lowering.o $(BUILDDIR)/llvm-late-gc-lowering.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-multiversioning.o $(BUILDDIR)/llvm-multiversioning.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-propagate-addrspaces.o $(BUILDDIR)/llvm-propagate-addrspaces.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-ptls.o $(BUILDDIR)/llvm-ptls.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h +$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h +$(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h + +$(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/processor.o $(BUILDDIR)/processor.dbg.obj: $(addprefix $(SRCDIR)/,processor_*.cpp processor.h features_*.h) $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/interpreter-stacktrace.c -$(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h -$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: \ - $(addprefix $(SRCDIR)/,debuginfo.h processor.h) -$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h -$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/staticdata.o $(BUILDDIR)/staticdata.dbg.obj: $(SRCDIR)/processor.h $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc.h diff --git a/src/anticodegen.c b/src/anticodegen.c index 30337f75d82e2..437607942bcc2 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -17,8 +17,10 @@ void jl_write_coverage_data(void) UNAVAILABLE JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) UNAVAILABLE -JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc, const char* asm_variant) UNAVAILABLE +JL_DLLEXPORT jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world, int raw_mc, char getwrapper, const char* asm_variant, const jl_cgparams_t params) UNAVAILABLE JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module) UNAVAILABLE +JL_DLLEXPORT void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params) UNAVAILABLE + JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, void *GetOpInfo, void *SymbolLookUp) UNAVAILABLE JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *Bytes, uint64_t BytesSize, uint64_t PC, char *OutString, size_t OutStringSize) UNAVAILABLE diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp new file mode 100644 index 0000000000000..d721b9b842ec6 --- /dev/null +++ b/src/aotcompile.cpp @@ -0,0 +1,441 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "llvm-version.h" +#include "platform.h" +#include "options.h" + +#if JL_LLVM_VERSION >= 40000 +# include +#else +# include +#endif +#include + +#include +#include +#include + +// target support +#include +#include +#include + +using namespace llvm; + +#include "julia.h" +#include "julia_internal.h" +#include "jitlayers.h" +#include "julia_assert.h" + +// MSVC's link.exe requires each function declaration to have a Comdat section +// So rather than litter the code with conditionals, +// all global values that get emitted call this function +// and it decides whether the definition needs a Comdat section and adds the appropriate declaration +template // for GlobalObject's +static T *addComdat(T *G) +{ +#if defined(_OS_WINDOWS_) + if (!G->isDeclaration()) { + // Add comdat information to make MSVC link.exe happy + // it's valid to emit this for ld.exe too, + // but makes it very slow to link for no benefit + if (G->getParent() == shadow_output) { +#if defined(_COMPILER_MICROSOFT_) + Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); + // ELF only supports Comdat::Any + jl_Comdat->setSelectionKind(Comdat::NoDuplicates); + G->setComdat(jl_Comdat); +#endif +#if defined(_CPU_X86_64_) + // Add unwind exception personalities to functions to handle async exceptions + assert(!juliapersonality_func || juliapersonality_func->getParent() == shadow_output); + if (Function *F = dyn_cast(G)) + F->setPersonalityFn(juliapersonality_func); +#endif + } + // add __declspec(dllexport) to everything marked for export + if (G->getLinkage() == GlobalValue::ExternalLinkage) + G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); + else + G->setDLLStorageClass(GlobalValue::DefaultStorageClass); + } +#endif + return G; +} + + +typedef struct { + std::unique_ptr M; + std::vector jl_sysimg_fvars; + std::vector jl_sysimg_gvars; + std::map> jl_fvar_map; + std::map jl_value_to_llvm; // uses 1-based indexing +} jl_native_code_desc_t; + +extern "C" +void jl_get_function_id(void *native_code, jl_method_instance_t *linfo, + uint8_t *api, uint32_t *func_idx, uint32_t *specfunc_idx) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) { + // get the function index in the fvar lookup table + auto it = data->jl_fvar_map.find(linfo); + if (it != data->jl_fvar_map.end()) { + std::tie(*api, *func_idx, *specfunc_idx) = it->second; + } + } +} + +extern "C" +int32_t jl_get_llvm_gv(void *native_code, jl_value_t *p) +{ + // map a jl_value_t memory location to a GlobalVariable + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) { + auto it = data->jl_value_to_llvm.find(p); + if (it != data->jl_value_to_llvm.end()) { + return it->second; + } + } + return 0; +} + +static void emit_offset_table(Module *mod, const std::vector &vars, StringRef name, Type *T_psize) +{ + // Emit a global variable with all the variable addresses. + // The cloning pass will convert them into offsets. + assert(!vars.empty()); + size_t nvars = vars.size(); + std::vector addrs(nvars); + for (size_t i = 0; i < nvars; i++) { + Constant *var = vars[i]; + addrs[i] = ConstantExpr::getBitCast(var, T_psize); + } + ArrayType *vars_type = ArrayType::get(T_psize, nvars); + new GlobalVariable(*mod, vars_type, true, + GlobalVariable::ExternalLinkage, + ConstantArray::get(vars_type, addrs), + name); +} + +static void jl_gen_llvm_globaldata(jl_native_code_desc_t *data, const char *sysimg_data, size_t sysimg_len) +{ + Module *mod = data->M.get(); + Type *T_size; + if (sizeof(size_t) == 8) + T_size = Type::getInt64Ty(mod->getContext()); + else + T_size = Type::getInt32Ty(mod->getContext()); + Type *T_psize = T_size->getPointerTo(); + emit_offset_table(mod, data->jl_sysimg_gvars, "jl_sysimg_gvars", T_psize); + emit_offset_table(mod, data->jl_sysimg_fvars, "jl_sysimg_fvars", T_psize); + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, globalUnique+1), + "jl_globalUnique")); + + // reflect the address of the jl_RTLD_DEFAULT_handle variable + // back to the caller, so that we can check for consistency issues + GlobalValue *jlRTLD_DEFAULT_var = mod->getNamedValue("jl_RTLD_DEFAULT_handle"); + addComdat(new GlobalVariable(*mod, + jlRTLD_DEFAULT_var->getType(), + true, + GlobalVariable::ExternalLinkage, + jlRTLD_DEFAULT_var, + "jl_RTLD_DEFAULT_handle_pointer")); + + if (sysimg_data) { + Constant *data = ConstantDataArray::get(mod->getContext(), + ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); + addComdat(new GlobalVariable(*mod, data->getType(), false, + GlobalVariable::ExternalLinkage, + data, "jl_system_image_data"))->setAlignment(64); + Constant *len = ConstantInt::get(T_size, sysimg_len); + addComdat(new GlobalVariable(*mod, len->getType(), true, + GlobalVariable::ExternalLinkage, + len, "jl_system_image_size")); + } +} + +static bool is_safe_char(unsigned char c) +{ + return ('0' <= c && c <= '9') || + ('A' <= c && c <= 'Z') || + ('a' <= c && c <= 'z') || + (c == '_' || c == '.') || + (c >= 128 && c < 255); +} + +static char hexchar(unsigned char c) +{ + return (c <= 9 ? '0' : 'A' - 10) + c; +} + +static const char *const common_names[255] = { +// 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x00 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x10 + 0, "NOT", 0, "YY", "XOR", 0, "AND", 0, 0, 0, "MUL", "SUM", 0, "SUB", 0, "DIV", // 0x20 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "COL", 0, "LT", "EQ", "GT", 0, // 0x30 + "AT", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "RDV", 0, "POW", 0, // 0x50 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "OR", 0, "TLD", 0, // 0x70 + 0 }; + +// removes special characters from the name of GlobalObjects, +// which might cause them to be treated special by LLVM +// or the system linker +// the only non-identifier characters we keep are '%' and '.', +// and all of UTF-8 above code-point 128 (except 255) +// most are given "friendly" abbreviations +// the remaining few will print as hex +static void makeSafeName(GlobalObject &G) +{ + StringRef Name = G.getName(); + SmallVector SafeName; + for (unsigned char c : Name.bytes()) { + if (is_safe_char(c)) + SafeName.push_back(c); + else { + SafeName.push_back('%'); + if (c == '%') { + SafeName.push_back('%'); + } + else if (common_names[c]) { + SafeName.push_back(common_names[c][0]); + SafeName.push_back(common_names[c][1]); + if (common_names[c][2]) + SafeName.push_back(common_names[c][2]); + } + else { + SafeName.push_back(hexchar((c >> 4) & 0xF)); + SafeName.push_back(hexchar(c & 0xF)); + } + } + } + if (SafeName.size() != Name.size()) + G.setName(StringRef(SafeName.data(), SafeName.size())); +} + + +// takes the running content that has collected in the shadow module and dump it to disk +// this builds the object file portion of the sysimage files for fast startup +extern "C" +void *jl_create_native(jl_array_t *methods) +{ + jl_native_code_desc_t *data = new jl_native_code_desc_t; + jl_codegen_params_t params; + std::map emitted; + jl_method_instance_t *mi = NULL; + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + JL_LOCK(&codegen_lock); + + for (int worlds = 2; worlds > 0; worlds--) { + params.world = (worlds == 1 ? jl_world_counter : jl_typeinf_world); + if (!params.world) + continue; + size_t i, l; + for (i = 0, l = jl_array_len(methods); i < l; i++) { + mi = (jl_method_instance_t*)jl_array_ptr_ref(methods, i); + if ((worlds == 1 || mi->max_world < jl_world_counter) && mi->min_world <= params.world && params.world <= mi->max_world) { + src = (jl_code_info_t*)mi->inferred; + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(mi->def.method, (jl_array_t*)src); + if (!src || !jl_is_code_info(src)) { + src = jl_type_infer(&mi, params.world, 0); + } + if (!emitted.count(mi)) { + jl_compile_result_t result = jl_compile_linfo1(mi, src, params); + if (std::get<0>(result)) + emitted[mi] = std::move(result); + } + } + } + jl_compile_workqueue(emitted, params); + } + JL_GC_POP(); + + // process the globals array, before jl_merge_module destroys them + std::vector gvars; + for (auto &global : params.globals) { + gvars.push_back(global.second->getName()); + data->jl_value_to_llvm[global.first] = gvars.size(); + } + + // clones the contents of the module `m` to the shadow_output collector + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(shadow_output, VMap)); + for (auto &def : emitted) { + jl_merge_module(clone.get(), std::move(std::get<0>(def.second))); + jl_method_instance_t *this_li = def.first; + jl_llvm_functions_t decls = std::get<1>(def.second); + jl_value_t *rettype = std::get<2>(def.second); + uint8_t api = std::get<3>(def.second); + Function *func = cast(clone->getNamedValue(decls.functionObject)); + Function *cfunc = NULL; + if (!decls.specFunctionObject.empty()) + cfunc = cast(clone->getNamedValue(decls.specFunctionObject)); + uint32_t func_id = 0; + uint32_t cfunc_id = 0; + if (cfunc && jl_egal(this_li->rettype, rettype)) { + data->jl_sysimg_fvars.push_back(cfunc); + cfunc_id = data->jl_sysimg_fvars.size(); + } + data->jl_sysimg_fvars.push_back(func); + func_id = data->jl_sysimg_fvars.size(); + data->jl_fvar_map[this_li] = std::make_tuple(api, func_id, cfunc_id); + } + + // now get references to the globals in the merged module + // and set them to be internalized and initialized at startup + for (auto &global : gvars) { + GlobalVariable *G = cast(clone->getNamedValue(global)); + G->setInitializer(ConstantPointerNull::get(cast(G->getValueType()))); + G->setLinkage(GlobalVariable::InternalLinkage); + data->jl_sysimg_gvars.push_back(G); + } + + // move everything inside, now that we've merged everything + // (before adding the exported headers) + for (GlobalObject &G : clone->global_objects()) { + if (!G.isDeclaration()) { + G.setLinkage(Function::InternalLinkage); + makeSafeName(G); + addComdat(&G); + } + } + + data->M = std::move(clone); + + JL_UNLOCK(&codegen_lock); // Might GC + return (void*)data; +} + + +// takes the running content that has collected in the shadow module and dump it to disk +// this builds the object file portion of the sysimage files for fast startup +extern "C" +void jl_dump_native(void *native_code, + const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, + const char *sysimg_data, size_t sysimg_len) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + JL_TIMING(NATIVE_DUMP); + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); + // make sure to emit the native object format, even if FORCE_ELF was set in codegen +#if defined(_OS_WINDOWS_) + TheTriple.setObjectFormat(Triple::COFF); +#elif defined(_OS_DARWIN_) + TheTriple.setObjectFormat(Triple::MachO); + TheTriple.setOS(llvm::Triple::MacOSX); +#endif + std::unique_ptr TM( + jl_TargetMachine->getTarget().createTargetMachine( + TheTriple.getTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + Reloc::PIC_, +#else + Optional(), +#endif + // Use small model so that we can use signed 32bits offset in the function and GV tables + CodeModel::Small, + CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? + )); + + legacy::PassManager PM; + addTargetPasses(&PM, TM.get()); + + // set up optimization passes + std::unique_ptr unopt_bc_OS; + std::unique_ptr bc_OS; + std::unique_ptr obj_OS; + + if (unopt_bc_fname) { + // call output handler directly to avoid special case handling of `-` filename + int FD; + std::error_code EC = sys::fs::openFileForWrite(unopt_bc_fname, FD, sys::fs::F_None); + unopt_bc_OS.reset(new raw_fd_ostream(FD, true)); + std::string err; + if (EC) + err = "ERROR: failed to open --output-unopt-bc file '" + std::string(unopt_bc_fname) + "': " + EC.message(); + if (!err.empty()) + jl_safe_printf("%s\n", err.c_str()); + else { + PM.add(createBitcodeWriterPass(*unopt_bc_OS.get())); + } + } + + if (bc_fname || obj_fname) + addOptimizationPasses(&PM, jl_options.opt_level, true); + + if (bc_fname) { + // call output handler directly to avoid special case handling of `-` filename + int FD; + std::error_code EC = sys::fs::openFileForWrite(bc_fname, FD, sys::fs::F_None); + bc_OS.reset(new raw_fd_ostream(FD, true)); + std::string err; + if (EC) + err = "ERROR: failed to open --output-bc file '" + std::string(bc_fname) + "': " + EC.message(); + if (!err.empty()) + jl_safe_printf("%s\n", err.c_str()); + else { + PM.add(createBitcodeWriterPass(*bc_OS.get())); + } + } + + if (obj_fname) { + // call output handler directly to avoid special case handling of `-` filename + int FD; + std::error_code EC = sys::fs::openFileForWrite(obj_fname, FD, sys::fs::F_None); + obj_OS.reset(new raw_fd_ostream(FD, true)); + std::string err; + if (EC) + err = "ERROR: failed to open --output-o file '" + std::string(obj_fname) + "': " + EC.message(); + if (!err.empty()) + jl_safe_printf("%s\n", err.c_str()); + else { + if (TM->addPassesToEmitFile(PM, *obj_OS.get(), TargetMachine::CGFT_ObjectFile, false)) { + jl_safe_printf("ERROR: target does not support generation of object files\n"); + } + } + } + + // Reset the target triple to make sure it matches the new target machine + data->M->setTargetTriple(TM->getTargetTriple().str()); +#if JL_LLVM_VERSION >= 40000 + DataLayout DL = TM->createDataLayout(); + DL.reset(DL.getStringRepresentation() + "-ni:10:11:12"); + data->M->setDataLayout(DL); +#else + data->M->setDataLayout(TM->createDataLayout()); +#endif + + // add metadata information + if (imaging_mode) + jl_gen_llvm_globaldata(data, sysimg_data, sysimg_len); + + // do the actual work + PM.run(*data->M); + imaging_mode = false; + + delete data; +} + +// clones the contents of the module `m` to the shadow_output collector +// TODO: this is deprecated +void jl_add_to_shadow(Module *m) +{ + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(m, VMap)); + jl_merge_module(shadow_output, std::move(clone)); +} diff --git a/src/ccall.cpp b/src/ccall.cpp index 3617c43ffbeaa..550f77bc153d8 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -41,6 +41,39 @@ lazyModule(Func &&func) std::forward(func)); } + +// global variables to pointers are pretty common, +// so this method is available as a convenience for emitting them. +// for other types, the formula for implementation is straightforward: +// (see stringConstPtr, for an alternative example to the code below) +// +// if in imaging_mode, emit a GlobalVariable with the same name and an initializer to the shadow_module +// making it valid for emission and reloading in the sysimage +// +// then add a global mapping to the current value (usually from calloc'd space) +// to the execution engine to make it valid for the current session (with the current value) +void* jl_emit_and_add_to_shadow(GlobalVariable *gv) +{ + // make a copy in the shadow_output + assert(imaging_mode); + PointerType *T = cast(gv->getValueType()); // pointer is the only supported type here + GlobalVariable *shadowvar = global_proto(gv, shadow_output); + shadowvar->setInitializer(ConstantPointerNull::get(T)); + shadowvar->setLinkage(GlobalVariable::InternalLinkage); + + // make the pointer valid for this session + void *slot = calloc(1, sizeof(void*)); + jl_ExecutionEngine->addGlobalMapping(gv, slot); + return slot; +} + +void* jl_get_globalvar(GlobalVariable *gv) +{ + void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(gv); + assert(p); + return p; +} + // Find or create the GVs for the library and symbol lookup. // Return `runtime_lib` (whether the library name is a string) // Optionally return the symbol address in the current session @@ -214,6 +247,8 @@ static DenseMap,GlobalVariable*>> allPltMap; +void jl_add_to_shadow(Module *m); + // Emit a "PLT" entry that will be lazily initialized // when being called the first time. static GlobalVariable *emit_plt_thunk( @@ -236,7 +271,6 @@ static GlobalVariable *emit_plt_thunk( Function *plt = Function::Create(functype, GlobalVariable::ExternalLinkage, fname, M); - jl_init_function(plt); plt->setAttributes(attrs); if (cc != CallingConv::C) plt->setCallingConv(cc); @@ -286,7 +320,8 @@ static GlobalVariable *emit_plt_thunk( } irbuilder.ClearInsertionPoint(); got = global_proto(got); // exchange got for the permanent global before jl_finalize_module destroys it - jl_finalize_module(M, true); + jl_add_to_shadow(M); + jl_finalize_module(std::unique_ptr(M)); auto shadowgot = cast(shadow_output->getNamedValue(gname)); @@ -1110,11 +1145,9 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar std::stringstream name; name << "jl_llvmcall" << llvmcallnumbering++; f->setName(name.str()); - jl_init_function(f); f = cast(prepare_call(function_proto(f))); } else { - jl_init_function(f); f->setLinkage(GlobalValue::LinkOnceODRLinkage); } @@ -1740,44 +1773,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) ctx.builder.SetInsertPoint(contBB); return ghostValue(jl_void_type); } - else if (is_libjulia_func(jl_function_ptr)) { - assert(!isVa && !llvmcall && nargt == 3); - assert(lrt == T_size); - jl_value_t *f = argv[0].constant; - jl_value_t *frt = argv[1].constant; - if (!frt) { - if (jl_is_type_type(argv[1].typ) && !jl_has_free_typevars(argv[1].typ)) - frt = jl_tparam0(argv[1].typ); - } - if (f && frt) { - jl_value_t *fargt = argv[2].constant;; - JL_GC_PUSH1(&fargt); - if (!fargt && jl_is_type_type(argv[2].typ)) { - if (!jl_has_free_typevars(argv[2].typ)) - fargt = jl_tparam0(argv[2].typ); - } - else if (fargt && jl_is_tuple(fargt)) { - // TODO: maybe deprecation warning, better checking - fargt = (jl_value_t*)jl_apply_tuple_type_v((jl_value_t**)jl_data_ptr(fargt), jl_nfields(fargt)); - } - if (fargt && jl_is_tuple_type(fargt)) { - Value *llvmf = NULL; - JL_TRY { - llvmf = jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt); - } - JL_CATCH { - llvmf = NULL; - } - if (llvmf) { - JL_GC_POP(); - JL_GC_POP(); - Value *fptr = ctx.builder.CreatePtrToInt(prepare_call(llvmf), lrt); - return mark_or_box_ccall_result(ctx, fptr, retboxed, rt, unionall, static_rt); - } - } - JL_GC_POP(); - } - } else if (is_libjulia_func(jl_array_isassigned) && argv[1].typ == (jl_value_t*)jl_ulong_type) { assert(!isVa && !llvmcall && nargt == 2 && !addressOf.at(0) && !addressOf.at(1)); @@ -1979,7 +1974,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } else if (symarg.fptr != NULL) { Type *funcptype = PointerType::get(functype, 0); - llvmf = literal_static_pointer_val(ctx, (void*)(uintptr_t)symarg.fptr, funcptype); + llvmf = literal_static_pointer_val((void*)(uintptr_t)symarg.fptr, funcptype); if (imaging_mode) jl_printf(JL_STDERR,"WARNING: literal address used in ccall for %s; code cannot be statically compiled\n", symarg.f_name); } @@ -2013,7 +2008,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } // since we aren't saving this code, there's no sense in // putting anything complicated here: just JIT the function address - llvmf = literal_static_pointer_val(ctx, symaddr, funcptype); + llvmf = literal_static_pointer_val(symaddr, funcptype); } } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 51eb820802343..4b7b85bfa23ec 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -240,30 +240,36 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) // --- emitting pointers directly into code --- -static Constant *literal_static_pointer_val(jl_codectx_t &ctx, const void *p, Type *T = T_pjlvalue) -{ - // this function will emit a static pointer into the generated code - // the generated code will only be valid during the current session, - // and thus, this should typically be avoided in new API's -#if defined(_P64) - return ConstantExpr::getIntToPtr(ConstantInt::get(T_int64, (uint64_t)p), T); -#else - return ConstantExpr::getIntToPtr(ConstantInt::get(T_int32, (uint32_t)p), T); -#endif -} +static inline Constant *literal_static_pointer_val(const void *p, Type *T = T_pjlvalue); static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) { - // emit a GlobalVariable for a jl_value_t named "cname" - return jl_get_global_for(cname, addr, jl_Module); + // first see if there already is a GlobalVariable for this address + GlobalVariable* &gv = ctx.global_targets[addr]; + Module *M = jl_Module; + if (!gv) { + // otherwise emit a new GlobalVariable for a jl_value_t named "cname" + std::stringstream gvname; + gvname << cname << globalUnique++; + // no existing GlobalVariable, create one and store it + gv = new GlobalVariable(*M, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); + } + else if (gv->getParent() != M) { + // re-use the same name, but move it to the new module + // this will help simplify merging them later + gv = prepare_global_in(M, gv); + } + return gv; } static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) { // emit a GlobalVariable for a jl_value_t, using the prefix, name, and module to // to create a readable name of the form prefixModA.ModB.name - size_t len = strlen(jl_symbol_name(name))+strlen(prefix)+1; + size_t len = strlen(jl_symbol_name(name)) + strlen(prefix) + 1; jl_module_t *parent = mod, *prev = NULL; while (parent != NULL && parent != prev) { len += strlen(jl_symbol_name(parent->name))+1; @@ -272,15 +278,14 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, j } char *fullname = (char*)alloca(len); strcpy(fullname, prefix); - int skipfirst = jl_symbol_name(name)[0] == '@'; - len -= strlen(jl_symbol_name(name)) + 1 - skipfirst; - strcpy(fullname + len, jl_symbol_name(name) + skipfirst); + len -= strlen(jl_symbol_name(name)) + 1; + strcpy(fullname + len, jl_symbol_name(name)); parent = mod; prev = NULL; while (parent != NULL && parent != prev) { - size_t part = strlen(jl_symbol_name(parent->name))+1-skipfirst; - strcpy(fullname+len-part,jl_symbol_name(parent->name)+skipfirst); - fullname[len-1] = '.'; + size_t part = strlen(jl_symbol_name(parent->name)) + 1; + strcpy(fullname + len - part, jl_symbol_name(parent->name)); + fullname[len - 1] = '.'; len -= part; prev = parent; parent = parent->parent; @@ -297,7 +302,7 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) Module *M = jl_Module; GlobalVariable *gv = new GlobalVariable( *M, T_pjlvalue, true, GlobalVariable::PrivateLinkage, - literal_static_pointer_val(ctx, p)); + literal_static_pointer_val(p)); gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); return gv; } @@ -388,7 +393,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) if (p == NULL) return V_null; if (!imaging_mode) - return literal_static_pointer_val(ctx, p); + return literal_static_pointer_val(p); Value *pgv = literal_pointer_val_slot(ctx, p); return tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable( ctx.builder.CreateLoad(T_pjlvalue, pgv), false, jl_typeof(p))); @@ -400,7 +405,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p) if (p == NULL) return V_null; if (!imaging_mode) - return literal_static_pointer_val(ctx, p); + return literal_static_pointer_val(p); // bindings are prefixed with jl_bnd# Value *pgv = julia_pgv(ctx, "jl_bnd#", p->name, p->owner, p); return tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable( @@ -447,7 +452,7 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b) ctx.builder.CreateLoad(T_pjlvalue, julia_pgv(ctx, "*", b->name, b->owner, b))), T_pprjlvalue); else - bv = ConstantExpr::getBitCast(literal_static_pointer_val(ctx, b), T_pprjlvalue); + bv = ConstantExpr::getBitCast(literal_static_pointer_val(b), T_pprjlvalue); return julia_binding_gv(ctx, bv); } @@ -2521,7 +2526,7 @@ static Value *emit_defer_signal(jl_codectx_t &ctx) static int compare_cgparams(const jl_cgparams_t *a, const jl_cgparams_t *b) { - return (a->cached == b->cached) && + return // language features (a->track_allocations == b->track_allocations) && (a->code_coverage == b->code_coverage) && diff --git a/src/codegen.cpp b/src/codegen.cpp index 5e1c1d4366e05..b3474c4e8353a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -86,7 +86,7 @@ #include #include #endif -#include +#include using namespace llvm; namespace llvm { @@ -158,22 +158,19 @@ extern void _chkstk(void); #define DISABLE_FLOAT16 -// llvm state -JL_DLLEXPORT LLVMContext jl_LLVMContext; -static bool nested_compile = false; -TargetMachine *jl_TargetMachine; - extern JITEventListener *CreateJuliaJITEventListener(); // for image reloading bool imaging_mode = false; +// shared llvm state +static LLVMContext jl_LLVMContext; +TargetMachine *jl_TargetMachine; Module *shadow_output; +static DataLayout jl_data_layout(""); #define jl_Module ctx.f->getParent() #define jl_builderModule(builder) (builder).GetInsertBlock()->getParent()->getParent() -static DataLayout jl_data_layout(""); - // types static Type *T_jlvalue; static Type *T_pjlvalue; @@ -502,27 +499,16 @@ typedef struct { jl_value_t *ty; } jl_arrayvar_t; -struct jl_returninfo_t { - Function *decl; - enum CallingConv { - Boxed = 0, - Register, - SRet, - Union, - Ghosts - } cc; - size_t union_bytes; - size_t union_align; - size_t union_minalign; -}; - -static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype); +static jl_returninfo_t get_specsig_function(Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype); // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. class jl_codectx_t { public: IRBuilder<> builder; + jl_codegen_params_t &emission_context; + jl_codegen_call_targets_t &call_targets; + std::map &global_targets; Function *f = NULL; // local var info. globals are not in here. std::vector slots; @@ -552,10 +538,17 @@ class jl_codectx_t { Value *world_age_field = NULL; bool debug_enabled = false; + bool use_cache = false; const jl_cgparams_t *params = NULL; - jl_codectx_t(LLVMContext &llvmctx) - : builder(llvmctx) { } + jl_codectx_t(LLVMContext &llvmctx, jl_codegen_params_t ¶ms) + : builder(llvmctx), + emission_context(params), + call_targets(params.workqueue), + global_targets(params.globals), + world(params.world), + use_cache(params.cache), + params(params.params) { } ~jl_codectx_t() { assert(this->roots == NULL); @@ -793,6 +786,32 @@ static void emit_write_barrier(jl_codectx_t&, Value*, Value*); #include "cgutils.cpp" +static void union_alloca_type(jl_uniontype_t *ut, + bool &allunbox, size_t &nbytes, size_t &align, size_t &min_align) +{ + nbytes = 0; + align = 0; + min_align = MAX_ALIGN; + // compute the size of the union alloca that could hold this type + unsigned counter = 0; + allunbox = for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + if (!jl_is_datatype_singleton(jt)) { + size_t nb1 = jl_datatype_size(jt); + size_t align1 = jl_datatype_align(jt); + if (nb1 > nbytes) + nbytes = nb1; + if (align1 > align) + align = align1; + if (align1 < min_align) + min_align = align1; + } + }, + (jl_value_t*)ut, + counter); +} + + static void jl_rethrow_with_add(const char *fmt, ...) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -1029,211 +1048,8 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return jl_cgval_t(v, typ, new_tindex); } -// Snooping on which functions are being compiled, and how long it takes -JL_STREAM *dump_compiles_stream = NULL; -uint64_t last_time = 0; -extern "C" JL_DLLEXPORT -void jl_dump_compiles(void *s) -{ - dump_compiles_stream = (JL_STREAM*)s; -} - // --- entry point --- //static int n_emit=0; -static std::unique_ptr emit_function( - jl_method_instance_t *lam, - jl_code_info_t *src, - size_t world, - jl_llvm_functions_t *declarations, - const jl_cgparams_t *params); -void jl_add_linfo_in_flight(StringRef name, jl_method_instance_t *linfo, const DataLayout &DL); - -const char *name_from_method_instance(jl_method_instance_t *li) -{ - return jl_is_method(li->def.method) ? jl_symbol_name(li->def.method->name) : "top-level scope"; -} - -// this generates llvm code for the lambda info -// and adds the result to the jitlayers -// (and the shadow module), but doesn't yet compile -// or generate object code for it -extern "C" -jl_llvm_functions_t jl_compile_linfo(jl_method_instance_t **pli, jl_code_info_t *src, size_t world, const jl_cgparams_t *params) -{ - // N.B.: `src` may have not been rooted by the caller. - JL_TIMING(CODEGEN); - jl_method_instance_t *li = *pli; - assert(jl_is_method_instance(li)); - jl_llvm_functions_t decls = {}; - - if (params != &jl_default_cgparams /* fast path */ && - !compare_cgparams(params, &jl_default_cgparams) && params->cached) - jl_error("functions compiled with custom codegen params mustn't be cached"); - - // Fast path for the already-compiled case - if (jl_is_method(li->def.method)) { - decls = li->functionObjectsDecls; - bool already_compiled = params->cached && decls.functionObject != NULL; - if (!src) { - if ((already_compiled || li->jlcall_api == JL_API_CONST) && - (li->min_world <= world && li->max_world >= world)) { - return decls; - } - } else if (already_compiled) { - return decls; - } - } - - JL_GC_PUSH1(&src); - JL_LOCK(&codegen_lock); - decls = li->functionObjectsDecls; - - // Codegen lock held in this block - { - // Step 1: Re-check if this was already compiled (it may have been while - // we waited at the lock). - if (!jl_is_method(li->def.method)) { - src = (jl_code_info_t*)li->inferred; - if (decls.functionObject != NULL || !src || !jl_is_code_info(src) || li->jlcall_api == JL_API_CONST) { - goto locked_out; - } - } - else if (!src) { - // If the caller didn't provide the source, - // try to infer it for ourself, but first, re-check if it's already compiled. - // assert(li->min_world <= world && li->max_world >= world); TODO: Re-enable at some point - if ((params->cached && decls.functionObject != NULL) || li->jlcall_api == JL_API_CONST) - goto locked_out; - - // see if it is inferred - src = (jl_code_info_t*)li->inferred; - if (src) { - if ((jl_value_t*)src != jl_nothing) - src = jl_uncompress_ast(li->def.method, (jl_array_t*)src); - if (!jl_is_code_info(src)) { - src = jl_type_infer(pli, world, 0); - li = *pli; - } - if (!src || li->jlcall_api == JL_API_CONST) - goto locked_out; - } - else { - // declare a failure to compile - goto locked_out; - } - } - else if (params->cached && decls.functionObject != NULL) { - // similar to above, but never returns a NULL - // decl (unless compile fails), even if jlcall_api == JL_API_CONST - goto locked_out; - } - else { - if ((jl_value_t*)src != jl_nothing) - src = jl_uncompress_ast(li->def.method, (jl_array_t*)src); - } - assert(jl_is_code_info(src)); - - // Step 2: setup global state - bool last_n_c = nested_compile; - if (!nested_compile && dump_compiles_stream != NULL) - last_time = jl_hrtime(); - nested_compile = true; - - // Step 3. actually do the work of emitting the function - std::unique_ptr m; - JL_TRY { - jl_llvm_functions_t *pdecls; - if (!params->cached) - pdecls = &decls; - else if (li->min_world <= world && li->max_world >= world) - pdecls = &li->functionObjectsDecls; - else if (!jl_is_method(li->def.method)) // toplevel thunk - pdecls = &li->functionObjectsDecls; - else - pdecls = &decls; - m = emit_function(li, src, world, pdecls, params); - if (params->cached && world) - decls = li->functionObjectsDecls; - //n_emit++; - } - JL_CATCH { - // something failed! this is very bad, since other WIP may be pointing to this function - // but there's not much we can do now. try to clear much of the WIP anyways. - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; - nested_compile = last_n_c; - JL_UNLOCK(&codegen_lock); // Might GC - const char *mname = name_from_method_instance(li); - jl_rethrow_with_add("error compiling %s", mname); - } - const char *f = decls.functionObject; - const char *specf = decls.specFunctionObject; - - if (JL_HOOK_TEST(params, module_activation)) { - JL_HOOK_CALL(params, module_activation, 1, jl_box_voidpointer(wrap(m.release()))); - } else { - // Step 4. Prepare debug info to receive this function - // record that this function name came from this linfo, - // so we can build a reverse mapping for debug-info. - bool toplevel = !jl_is_method(li->def.method); - if (!toplevel) { - const DataLayout &DL = m->getDataLayout(); - // but don't remember toplevel thunks because - // they may not be rooted in the gc for the life of the program, - // and the runtime doesn't notify us when the code becomes unreachable :( - jl_add_linfo_in_flight(StringRef(specf ? specf : f), li, DL); - } - - // Step 5. Add the result to the execution engine now - jl_finalize_module(m.release(), !toplevel); - } - - // if not inlineable, code won't be needed again - if (JL_DELETE_NON_INLINEABLE && - // don't delete code when debugging level >= 2 - jl_options.debug_level <= 1 && - // don't delete toplevel code - jl_is_method(li->def.method) && - // don't change inferred state - li->inferred && - // and there is something to delete (test this before calling jl_ast_flag_inlineable) - li->inferred != jl_nothing && - // don't delete inlineable code, unless it is constant - (li->jlcall_api == JL_API_CONST || !jl_ast_flag_inlineable((jl_array_t*)li->inferred)) && - // don't delete code when generating a precompile file - !imaging_mode && - // don't delete code when it's not actually directly being used - world) { - li->inferred = jl_nothing; - } - - // Step 6: Done compiling: Restore global state - nested_compile = last_n_c; - } - - JL_UNLOCK(&codegen_lock); // Might GC - - // If logging of the compilation stream is enabled then dump the function to the stream - // ... unless li->def isn't defined here meaning the function is a toplevel thunk and - // would have its CodeInfo printed in the stream, which might contain double-quotes that - // would not be properly escaped given the double-quotes added to the stream below. - if (dump_compiles_stream != NULL && jl_is_method(li->def.method)) { - uint64_t this_time = jl_hrtime(); - jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", this_time - last_time); - jl_static_show(dump_compiles_stream, li->specTypes); - jl_printf(dump_compiles_stream, "\"\n"); - last_time = this_time; - } - JL_GC_POP(); - return decls; - -locked_out: - JL_UNLOCK(&codegen_lock); - JL_GC_POP(); - return decls; -} - -#define getModuleFlag(m,str) m->getModuleFlag(str) static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default_cgparams) { @@ -1245,7 +1061,7 @@ static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default // Some linkers (*cough* OS X) don't understand DWARF v4, so we use v2 in // imaging mode. The structure of v4 is slightly nicer for debugging JIT // code. - if (!getModuleFlag(m,"Dwarf Version")) { + if (!m->getModuleFlag("Dwarf Version")) { int dwarf_version = 4; #ifdef _OS_DARWIN_ if (imaging_mode) @@ -1253,7 +1069,7 @@ static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default #endif m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", dwarf_version); } - if (!getModuleFlag(m,"Debug Info Version")) + if (!m->getModuleFlag("Debug Info Version")) m->addModuleFlag(llvm::Module::Error, "Debug Info Version", llvm::DEBUG_METADATA_VERSION); #if JL_LLVM_VERSION >= 40000 @@ -1265,345 +1081,67 @@ static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default } -// this ensures that llvmf has been emitted to the execution engine, -// returning the function pointer to it -extern void jl_callback_triggered_linfos(void); -static uint64_t getAddressForFunction(StringRef fname) -{ - JL_TIMING(LLVM_EMIT); -#ifdef JL_DEBUG_BUILD - llvm::raw_fd_ostream out(1, false); -#endif - jl_finalize_function(fname); - uint64_t ret = jl_ExecutionEngine->getFunctionAddress(fname); - // delay executing trace callbacks until here to make sure there's no - // recursive compilation. - jl_callback_triggered_linfos(); - return ret; -} - -// convenience helper exported for usage from gdb -extern "C" JL_DLLEXPORT -uint64_t jl_get_llvm_fptr(void *function) +static std::pair uses_specsig(jl_method_instance_t *lam, bool prefer_specsig) { - Function *F = (Function*)function; - uint64_t addr = getAddressForFunction(F->getName()); - if (!addr) { -#if JL_LLVM_VERSION >= 50000 - if (auto exp_addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress()) { - addr = exp_addr.get(); - } -#else - addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress(); -#endif + size_t nreq = jl_is_method(lam->def.method) ? lam->def.method->nargs : 0; + int va = 0; + if (nreq > 0 && lam->def.method->isva) { + nreq--; + va = 1; } - return addr; -} + jl_value_t *sig = lam->specTypes; + jl_value_t *rettype = lam->rettype; -static jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method) -{ - // one unspecialized version of a function can be shared among all cached specializations - jl_method_t *def = method->def.method; - if (def->source == NULL) { - return method; - } - if (def->unspecialized == NULL) { - JL_LOCK(&def->writelock); - if (def->unspecialized == NULL) { - def->unspecialized = jl_get_specialized(def, def->sig, jl_emptysvec); - jl_gc_wb(def, def->unspecialized); + bool needsparams = false; + if (jl_is_method(lam->def.method)) { + if (jl_svec_len(lam->def.method->sparam_syms) != jl_svec_len(lam->sparam_vals)) + needsparams = true; + for (size_t i = 0; i < jl_svec_len(lam->sparam_vals); ++i) { + if (jl_is_typevar(jl_svecref(lam->sparam_vals, i))) + needsparams = true; } - JL_UNLOCK(&def->writelock); } - return def->unspecialized; -} -// this compiles li and emits fptr -extern "C" -jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t *li, const char *F, size_t world) -{ - jl_generic_fptr_t fptr; - fptr.fptr = li->fptr; - fptr.jlcall_api = li->jlcall_api; - if (fptr.fptr && fptr.jlcall_api) { - return fptr; - } - fptr.fptr = li->unspecialized_ducttape; - fptr.jlcall_api = JL_API_GENERIC; - if (!li->inferred && fptr.fptr) { - return fptr; - } - JL_LOCK(&codegen_lock); - fptr.fptr = li->fptr; - fptr.jlcall_api = li->jlcall_api; - if (fptr.fptr && fptr.jlcall_api) { - JL_UNLOCK(&codegen_lock); - return fptr; - } - jl_method_instance_t *unspec = NULL; - if (jl_is_method(li->def.method)) { - if (li->def.method->unspecialized) { - unspec = li->def.method->unspecialized; - } - if (!F || !jl_can_finalize_function(F)) { - // can't compile F in the JIT right now, - // so instead compile an unspecialized version - // and return its fptr instead - if (!unspec) - unspec = jl_get_unspecialized(li); // get-or-create the unspecialized version to cache the result - jl_code_info_t *src = (jl_code_info_t*)unspec->def.method->source; - if (src == NULL) { - assert(unspec->def.method->generator); - src = jl_code_for_staged(unspec); - } - fptr.fptr = unspec->fptr; - fptr.jlcall_api = unspec->jlcall_api; - if (fptr.fptr && fptr.jlcall_api) { - JL_UNLOCK(&codegen_lock); - return fptr; - } - jl_llvm_functions_t decls = unspec->functionObjectsDecls; - if (unspec == li) { - // temporarily clear the decls so that it will compile our unspec version of src - unspec->functionObjectsDecls.functionObject = NULL; - unspec->functionObjectsDecls.specFunctionObject = NULL; - } - assert(src); - F = jl_compile_linfo(&unspec, src, unspec->min_world, &jl_default_cgparams).functionObject; // this does not change unspec - if (unspec == li) { - unspec->functionObjectsDecls = decls; - } - assert(jl_can_finalize_function(F)); + if (needsparams) + return std::make_pair(false, true); + if (sig == (jl_value_t*)jl_anytuple_type) + return std::make_pair(false, false); + if (!jl_is_datatype(sig)) + return std::make_pair(false, false); + if (jl_nparams(sig) == 0) + return std::make_pair(false, false); + if (va) { + if (jl_is_vararg_type(jl_tparam(sig, jl_nparams(sig) - 1))) + return std::make_pair(false, false); + // For now we can only handle va tuples that will end up being + // leaf types + for (size_t i = nreq; i < jl_nparams(sig); i++) { + if (!jl_justbits(jl_tparam(sig, i))) + return std::make_pair(false, false); } } - assert(F); - fptr.fptr = (jl_fptr_t)getAddressForFunction(F); - fptr.jlcall_api = jl_jlcall_api(F); - assert(fptr.fptr != NULL); - // decide if the fptr should be cached somewhere also - if (li->functionObjectsDecls.functionObject == F) { - if (li->fptr) { - // don't change fptr as that leads to race conditions - // with the (not) simultaneous update to jlcall_api - } - else if (li->inferred || fptr.jlcall_api != JL_API_GENERIC) { - li->jlcall_api = fptr.jlcall_api; - li->fptr = fptr.fptr; - } - else { - li->unspecialized_ducttape = fptr.fptr; - } + // not invalid, consider if specialized signature is worthwhile + if (prefer_specsig) + return std::make_pair(true, false); + if (jl_justbits(rettype) && !jl_is_datatype_singleton((jl_datatype_t*)rettype)) + return std::make_pair(true, false); + if (jl_is_uniontype(rettype)) { + bool allunbox; + size_t nbytes, align, min_align; + union_alloca_type((jl_uniontype_t*)rettype, allunbox, nbytes, align, min_align); + if (nbytes > 0) + return std::make_pair(true, false); // some elements of the union could be returned unboxed avoiding allocation } - else if (unspec) { - if (unspec->fptr) { - // don't change fptr as that leads to race conditions - // with the (not) simultaneous update to jlcall_api - } - else if (unspec == li) { - if (fptr.jlcall_api == JL_API_GENERIC) - li->unspecialized_ducttape = fptr.fptr; - } - else if (unspec->functionObjectsDecls.functionObject == F) { - unspec->jlcall_api = fptr.jlcall_api; - unspec->fptr = fptr.fptr; + for (size_t i = 0; i < jl_nparams(sig); i++) { + jl_value_t *sigt = jl_tparam(sig, i); + if (jl_justbits(sigt) && !jl_is_datatype_singleton((jl_datatype_t*)sigt)) { + return std::make_pair(true, false); } } - JL_UNLOCK(&codegen_lock); // Might GC - return fptr; -} - -static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt); -// get the address of a C-callable entry point for a function -extern "C" JL_DLLEXPORT -void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) -{ - JL_GC_PUSH1(&argt); - JL_LOCK(&codegen_lock); - Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - JL_GC_POP(); - void *ptr = (void*)getAddressForFunction(llvmf->getName()); - JL_UNLOCK(&codegen_lock); - return ptr; -} - - -// convenience function for debugging from gdb (pre-OrcJIT) -// it generally helps to have define KEEP_BODIES if you plan on using this -extern "C" JL_DLLEXPORT -void *jl_function_ptr_by_llvm_name(char *name) { -#ifdef JL_MSAN_ENABLED - __msan_unpoison_string(name); -#endif - return (void*)jl_ExecutionEngine->FindFunctionNamed(name); // returns an llvm::Function* -} - -// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name -extern "C" JL_DLLEXPORT -void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) -{ - assert(jl_is_tuple_type(argt)); - JL_LOCK(&codegen_lock); - Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) - uint64_t Addr = getAddressForFunction(llvmf->getName()); - - if (imaging_mode) - llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); - - // make the alias to the shadow_module - GlobalAlias *GA = - GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, name, llvmf, shadow_output); - - // make sure the alias name is valid for the current session - jl_ExecutionEngine->addGlobalMapping(GA, (void*)(uintptr_t)Addr); - JL_UNLOCK(&codegen_lock); + return std::make_pair(false, false); // jlcall sig won't require any box allocations } -// --- native code info, and dump function to IR and ASM --- -// Get pointer to llvm::Function instance, compiling if necessary -// for use in reflection from Julia. -// this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: -// misuse will leak memory or cause read-after-free -extern "C" JL_DLLEXPORT -void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, bool getwrapper, bool optimize, const jl_cgparams_t params) -{ - if (jl_is_method(linfo->def.method) && linfo->def.method->source == NULL && - linfo->def.method->generator == NULL) { - // not a generic function - return NULL; - } - - jl_code_info_t *src = (jl_code_info_t*)linfo->inferred; - JL_GC_PUSH1(&src); - if (!src || (jl_value_t*)src == jl_nothing) { - src = jl_type_infer(&linfo, world, 0); - if (!src && jl_is_method(linfo->def.method)) - src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source; - } - if ((jl_value_t*)src == jl_nothing) - src = NULL; - if (src && !jl_is_code_info(src) && jl_is_method(linfo->def.method)) - src = jl_uncompress_ast(linfo->def.method, (jl_array_t*)src); - if (src && !jl_is_code_info(src)) - src = NULL; - if (!src) - jl_error("source not found for function"); - // Backup the info for the nested compile - JL_LOCK(&codegen_lock); - - bool last_n_c = nested_compile; - nested_compile = true; - // emit this function into a new module - jl_llvm_functions_t declarations; - std::unique_ptr m; - JL_TRY { - m = emit_function(linfo, src, world, &declarations, ¶ms); - } - JL_CATCH { - // something failed! - nested_compile = last_n_c; - JL_UNLOCK(&codegen_lock); // Might GC - const char *mname = name_from_method_instance(linfo); - jl_rethrow_with_add("error compiling %s", mname); - } - // Restore the previous compile context - nested_compile = last_n_c; - - if (optimize) - jl_globalPM->run(*m.get()); - const char *fname = declarations.functionObject; - const char *specfname = declarations.specFunctionObject; - Function *f = NULL; - Function *specf = NULL; - // swap declarations for definitions and destroy declarations - if (specfname) { - specf = cast(m->getNamedValue(specfname)); - free(const_cast(specfname)); - } - if (fname) { - f = cast(m->getNamedValue(fname)); - free(const_cast(fname)); - } - // clone the name from the runtime linfo, if it exists - // to give the user a (false) sense of stability - specfname = linfo->functionObjectsDecls.specFunctionObject; - if (specfname) { - specf->setName(specfname); - } - fname = linfo->functionObjectsDecls.functionObject; - if (fname) { - f->setName(fname); - } - m.release(); // the return object `llvmf` will be the owning pointer - JL_UNLOCK(&codegen_lock); // Might GC - JL_GC_POP(); - if (getwrapper || !specf) - return f; - else - return specf; -} - - -extern "C" JL_DLLEXPORT -void *jl_get_llvmf_decl(jl_method_instance_t *linfo, size_t world, bool getwrapper, const jl_cgparams_t params) -{ - if (jl_is_method(linfo->def.method) && linfo->def.method->source == NULL && - linfo->def.method->generator == NULL) { - // not a generic function - return NULL; - } - - // compile this normally - jl_code_info_t *src = NULL; - if (linfo->inferred == NULL) - src = jl_type_infer(&linfo, world, 0); - jl_llvm_functions_t decls = jl_compile_linfo(&linfo, src, world, &jl_default_cgparams); - - if (decls.functionObject == NULL && linfo->jlcall_api == JL_API_CONST && jl_is_method(linfo->def.method)) { - // normally we don't generate native code for these functions, so need an exception here - // This leaks a bit of memory to cache native code that we'll never actually need - JL_LOCK(&codegen_lock); - decls = linfo->functionObjectsDecls; - if (decls.functionObject == NULL) { - src = jl_type_infer(&linfo, world, 0); - if (!src) { - src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source; - } - decls = jl_compile_linfo(&linfo, src, world, ¶ms); - linfo->functionObjectsDecls = decls; - } - JL_UNLOCK(&codegen_lock); - } - - if (getwrapper || !decls.specFunctionObject) { - auto f = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, decls.functionObject); - add_return_attr(f, Attribute::NonNull); - f->addFnAttr("thunk"); - return f; - } - else { - jl_returninfo_t returninfo = get_specsig_function(NULL, decls.specFunctionObject, linfo->specTypes, linfo->rettype); - return returninfo.decl; - } -} - -// get a native disassembly for f (an LLVM function) -// warning: this takes ownership of, and destroys, f -extern "C" JL_DLLEXPORT -const jl_value_t *jl_dump_function_asm(void *f, int raw_mc, const char* asm_variant) -{ - Function *llvmf = dyn_cast_or_null((Function*)f); - if (!llvmf) - jl_error("jl_dump_function_asm: Expected Function*"); - uint64_t fptr = getAddressForFunction(llvmf->getName()); - // Look in the system image as well - if (fptr == 0) - fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(llvmf); - delete llvmf; - return jl_dump_fptr_asm(fptr, raw_mc, asm_variant); -} // Logging for code coverage and memory allocation @@ -2898,106 +2436,111 @@ static Value *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, } -static jl_cgval_t emit_call_function_object(jl_method_instance_t *li, jl_llvm_functions_t decls, - jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty, jl_codectx_t &ctx) +static jl_cgval_t emit_call_specsig_object(jl_codectx_t &ctx, jl_method_instance_t *li, StringRef specFunctionObject, + jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, + jl_value_t *inferred_retty) { - if (decls.specFunctionObject != NULL) { - // emit specialized call site - jl_value_t *jlretty = li->rettype; - jl_returninfo_t returninfo = get_specsig_function(jl_Module, decls.specFunctionObject, li->specTypes, jlretty); - FunctionType *cft = returninfo.decl->getFunctionType(); + // emit specialized call site + jl_value_t *jlretty = li->rettype; + jl_returninfo_t returninfo = get_specsig_function(jl_Module, specFunctionObject, li->specTypes, jlretty); + FunctionType *cft = returninfo.decl->getFunctionType(); + *cc = returninfo.cc; + + size_t nfargs = cft->getNumParams(); + Value **argvals = (Value**)alloca(nfargs * sizeof(Value*)); + unsigned idx = 0; + AllocaInst *result; + switch (returninfo.cc) { + case jl_returninfo_t::Boxed: + case jl_returninfo_t::Register: + case jl_returninfo_t::Ghosts: + break; + case jl_returninfo_t::SRet: + result = emit_static_alloca(ctx, cft->getParamType(0)->getContainedType(0)); + argvals[idx] = decay_derived(result); + idx++; + break; + case jl_returninfo_t::Union: + result = emit_static_alloca(ctx, ArrayType::get(T_int8, returninfo.union_bytes)); + if (returninfo.union_align > 1) + result->setAlignment(returninfo.union_align); + argvals[idx] = result; + idx++; + break; + } - size_t nfargs = cft->getNumParams(); - Value **argvals = (Value**)alloca(nfargs * sizeof(Value*)); - unsigned idx = 0; - AllocaInst *result; - switch (returninfo.cc) { + for (size_t i = 0; i < nargs; i++) { + jl_value_t *jt = jl_nth_slot_type(li->specTypes, i); + bool isboxed; + Type *et = julia_type_to_llvm(jt, &isboxed); + if (type_is_ghost(et)) + continue; + assert(idx < nfargs); + Type *at = cft->getParamType(idx); + const jl_cgval_t &arg = argv[i]; + if (isboxed) { + assert(at == T_prjlvalue && (et == T_pjlvalue || et == T_prjlvalue)); + argvals[idx] = maybe_decay_untracked(boxed(ctx, arg)); + } + else if (et->isAggregateType()) { + // can lazy load on demand, no copy needed + assert(at == PointerType::get(et, AddressSpace::Derived)); + assert(arg.ispointer()); + argvals[idx] = decay_derived(maybe_bitcast(ctx, + data_pointer(ctx, arg), at)); + } + else { + assert(at == et); + argvals[idx] = emit_unbox(ctx, et, arg, jt); + } + idx++; + } + assert(idx == nfargs); + CallInst *call = ctx.builder.CreateCall(returninfo.decl, ArrayRef(&argvals[0], nfargs)); + call->setAttributes(returninfo.decl->getAttributes()); + + jl_cgval_t retval; + switch (returninfo.cc) { case jl_returninfo_t::Boxed: + retval = mark_julia_type(ctx, call, true, inferred_retty); + break; case jl_returninfo_t::Register: - case jl_returninfo_t::Ghosts: + retval = mark_julia_type(ctx, call, false, jlretty); break; case jl_returninfo_t::SRet: - result = emit_static_alloca(ctx, cft->getParamType(0)->getContainedType(0)); - argvals[idx] = decay_derived(result); - idx++; + retval = mark_julia_slot(result, jlretty, NULL, tbaa_stack); break; - case jl_returninfo_t::Union: - result = emit_static_alloca(ctx, ArrayType::get(T_int8, returninfo.union_bytes)); - if (returninfo.union_align > 1) - result->setAlignment(returninfo.union_align); - argvals[idx] = result; - idx++; + case jl_returninfo_t::Union: { + Value *box = ctx.builder.CreateExtractValue(call, 0); + Value *tindex = ctx.builder.CreateExtractValue(call, 1); + Value *derived = ctx.builder.CreateSelect( + ctx.builder.CreateICmpEQ( + ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)), + decay_derived(ctx.builder.CreateBitCast(argvals[0], T_pjlvalue)), + decay_derived(box) + ); + retval = mark_julia_slot(derived, + jlretty, + tindex, + tbaa_stack); + retval.Vboxed = box; break; } - - for (size_t i = 0; i < nargs; i++) { - jl_value_t *jt = jl_nth_slot_type(li->specTypes, i); - bool isboxed; - Type *et = julia_type_to_llvm(jt, &isboxed); - if (type_is_ghost(et)) - continue; - assert(idx < nfargs); - Type *at = cft->getParamType(idx); - const jl_cgval_t &arg = argv[i]; - if (isboxed) { - assert(at == T_prjlvalue && (et == T_pjlvalue || et == T_prjlvalue)); - argvals[idx] = maybe_decay_untracked(boxed(ctx, arg)); - } - else if (et->isAggregateType()) { - // can lazy load on demand, no copy needed - assert(at == PointerType::get(et, AddressSpace::Derived)); - assert(arg.ispointer()); - argvals[idx] = decay_derived(maybe_bitcast(ctx, - data_pointer(ctx, arg), at)); - } - else { - assert(at == et); - argvals[idx] = emit_unbox(ctx, et, arg, jt); - } - idx++; - } - assert(idx == nfargs); - CallInst *call = ctx.builder.CreateCall(returninfo.decl, ArrayRef(&argvals[0], nfargs)); - call->setAttributes(returninfo.decl->getAttributes()); - - jl_cgval_t retval; - switch (returninfo.cc) { - case jl_returninfo_t::Boxed: - retval = mark_julia_type(ctx, call, true, inferred_retty); - break; - case jl_returninfo_t::Register: - retval = mark_julia_type(ctx, call, false, jlretty); - break; - case jl_returninfo_t::SRet: - retval = mark_julia_slot(result, jlretty, NULL, tbaa_stack); - break; - case jl_returninfo_t::Union: { - Value *box = ctx.builder.CreateExtractValue(call, 0); - Value *tindex = ctx.builder.CreateExtractValue(call, 1); - Value *derived = ctx.builder.CreateSelect( - ctx.builder.CreateICmpEQ( - ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), - ConstantInt::get(T_int8, 0)), - decay_derived(ctx.builder.CreateBitCast(argvals[0], T_pjlvalue)), - decay_derived(box) - ); - retval = mark_julia_slot(derived, - jlretty, - tindex, - tbaa_stack); - retval.Vboxed = box; - break; - } - case jl_returninfo_t::Ghosts: - retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); - break; - } - // see if inference has a different / better type for the call than the lambda - if (inferred_retty != retval.typ) - retval = update_julia_type(ctx, retval, inferred_retty); - return retval; + case jl_returninfo_t::Ghosts: + retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); + break; } - auto theFptr = jl_Module->getOrInsertFunction(decls.functionObject, jl_func_sig); + // see if inference has a different / better type for the call than the lambda + if (inferred_retty != retval.typ) + retval = update_julia_type(ctx, retval, inferred_retty); + return retval; +} + +static jl_cgval_t emit_call_function_object(jl_codectx_t &ctx, jl_method_instance_t *li, StringRef functionObject, + jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) +{ + auto theFptr = jl_Module->getOrInsertFunction(functionObject, jl_func_sig); if (auto F = dyn_cast(theFptr->stripPointerCasts())) { add_return_attr(F, Attribute::NonNull); F->addFnAttr("thunk"); @@ -3020,32 +2563,83 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex) argv[i] = emit_expr(ctx, args[i + 1]); } + jl_cgval_t result; + jl_method_instance_t *li = NULL; if (lival.constant) { - jl_method_instance_t *li = (jl_method_instance_t*)lival.constant; + li = (jl_method_instance_t*)lival.constant; assert(jl_is_method_instance(li)); - jl_llvm_functions_t decls = jl_compile_linfo(&li, NULL, ctx.world, ctx.params); + assert((li->min_world <= ctx.linfo->min_world && li->max_world >= ctx.linfo->max_world) || + (ctx.linfo->min_world == 0 && ctx.linfo->max_world == 0)); if (li->jlcall_api == JL_API_CONST) { assert(li->inferred_const); return mark_julia_const(li->inferred_const); } - if (decls.functionObject) { - int jlcall_api = jl_jlcall_api(decls.functionObject); - if (jlcall_api == JL_API_GENERIC) { - jl_cgval_t result = emit_call_function_object(li, decls, argv, nargs, rt, ctx); - if (result.typ == jl_bottom_type) - CreateTrap(ctx.builder); - return result; + if (li == ctx.linfo) { + // handle self-recursion specially + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; + FunctionType *ft = ctx.f->getFunctionType(); + StringRef protoname = ctx.f->getName(); + if (ft == jl_func_sig) + result = emit_call_function_object(ctx, li, protoname, argv, nargs, rt); + else if (ft != jl_func_sig_sparams) + result = emit_call_specsig_object(ctx, li, protoname, argv, nargs, &cc, rt); + else + li = NULL; // fallback to dispatch + } + else if (li->jlcall_api == JL_API_GENERIC || li->jlcall_api == JL_API_NOT_SET) { + bool specsig, needsparams; + std::tie(specsig, needsparams) = uses_specsig(li, ctx.params->prefer_specsig); + if (!jl_is_rettype_inferred(li)) + specsig = false; + std::string name; + StringRef protoname; + bool need_to_emit = true; + if (ctx.use_cache) { + // optimization: emit the correct name immediately, if we know it + // TODO: use `emitted` map here too to try to consolidate names? + if (specsig) { + if (li->fptr_specsig) { + protoname = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)li->fptr_specsig, li); + need_to_emit = false; + } + } + else { + if (li->fptr && li->jlcall_api == JL_API_GENERIC) { + protoname = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)li->fptr, li); + need_to_emit = false; + } + } + } + if (need_to_emit) { + std::stringstream namestream; + namestream << (specsig ? "julia_spectrampoline_" : "julia_trampoline_") << globalUnique++; + name = namestream.str(); + protoname = StringRef(name); } + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; + if (specsig) + result = emit_call_specsig_object(ctx, li, protoname, argv, nargs, &cc, rt); + else + result = emit_call_function_object(ctx, li, protoname, argv, nargs, rt); + if (need_to_emit) { + Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); + ctx.call_targets.push_back(std::make_tuple(li, cc, trampoline_decl, li->rettype, specsig)); + } + } + else { + li = NULL; } } - jl_cgval_t result = mark_julia_type(ctx, - emit_jlcall( - ctx, - prepare_call(jlinvoke_func), - boxed(ctx, lival), - argv, nargs), - true, - rt); + if (!li) { + result = mark_julia_type(ctx, + emit_jlcall( + ctx, + prepare_call(jlinvoke_func), + boxed(ctx, lival), + argv, nargs), + true, + rt); + } if (result.typ == jl_bottom_type) CreateTrap(ctx.builder); return result; @@ -3372,31 +2966,6 @@ static jl_cgval_t emit_local(jl_codectx_t &ctx, jl_value_t *slotload) } -static void union_alloca_type(jl_uniontype_t *ut, - bool &allunbox, size_t &nbytes, size_t &align, size_t &min_align) -{ - nbytes = 0; - align = 0; - min_align = MAX_ALIGN; - // compute the size of the union alloca that could hold this type - unsigned counter = 0; - allunbox = for_each_uniontype_small( - [&](unsigned idx, jl_datatype_t *jt) { - if (!jl_is_datatype_singleton(jt)) { - size_t nb1 = jl_datatype_size(jt); - size_t align1 = jl_datatype_align(jt); - if (nb1 > nbytes) - nbytes = nb1; - if (align1 > align) - align = align1; - if (align1 < min_align) - min_align = align1; - } - }, - (jl_value_t*)ut, - counter); -} - static Value *try_emit_union_alloca(jl_codectx_t &ctx, jl_uniontype_t *ut, bool &allunbox, size_t &min_align) { size_t nbytes, align; @@ -4016,12 +3585,12 @@ static void emit_last_age_field(jl_codectx_t &ctx) static void emit_cfunc_invalidate( Function *gf_thunk, jl_returninfo_t::CallingConv cc, - jl_method_instance_t *lam, size_t nargs, size_t world) + jl_value_t *calltype, jl_value_t *rettype, + size_t nargs, + jl_codegen_params_t ¶ms) { - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = gf_thunk; - ctx.world = world; - ctx.params = &jl_default_cgparams; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", gf_thunk); ctx.builder.SetInsertPoint(b0); @@ -4034,7 +3603,7 @@ static void emit_cfunc_invalidate( if (cc == jl_returninfo_t::SRet || cc == jl_returninfo_t::Union) ++AI; for (size_t i = 0; i < nargs; i++) { - jl_value_t *jt = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *jt = jl_nth_slot_type(calltype, i); bool isboxed; Type *et = julia_type_to_llvm(jt, &isboxed); if (type_is_ghost(et)) { @@ -4062,9 +3631,8 @@ static void emit_cfunc_invalidate( assert(AI == gf_thunk->arg_end()); Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, myargs, nargs); jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type); - jl_value_t *astrt = lam->rettype; if (cc != jl_returninfo_t::Boxed) { - emit_typecheck(ctx, gf_retbox, astrt, "cfunction"); + emit_typecheck(ctx, gf_retbox, rettype, "cfunction"); } switch (cc) { @@ -4078,12 +3646,12 @@ static void emit_cfunc_invalidate( } else { gf_ret = emit_bitcast(ctx, gf_ret, gfrt->getPointerTo()); - ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, julia_alignment(astrt, 0))); + ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, julia_alignment(rettype, 0))); } break; } case jl_returninfo_t::SRet: { - unsigned sret_nbytes = jl_datatype_size(astrt); + unsigned sret_nbytes = jl_datatype_size(rettype); emit_memcpy(ctx, &*gf_thunk->arg_begin(), gf_ret, sret_nbytes, jl_alignment(sret_nbytes)); ctx.builder.CreateRetVoid(); break; @@ -4091,7 +3659,7 @@ static void emit_cfunc_invalidate( case jl_returninfo_t::Union: { Type *retty = gf_thunk->getReturnType(); Value *gf_retval = UndefValue::get(retty); - Value *tindex = compute_box_tindex(ctx, gf_ret, (jl_value_t*)jl_any_type, astrt); + Value *tindex = compute_box_tindex(ctx, gf_ret, (jl_value_t*)jl_any_type, rettype); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); gf_retval = ctx.builder.CreateInsertValue(gf_retval, gf_ret, 0); gf_retval = ctx.builder.CreateInsertValue(gf_retval, tindex, 1); @@ -4099,7 +3667,7 @@ static void emit_cfunc_invalidate( break; } case jl_returninfo_t::Ghosts: { - Value *gf_retval = compute_tindex_unboxed(ctx, gf_retbox, astrt); + Value *gf_retval = compute_tindex_unboxed(ctx, gf_retbox, rettype); ctx.builder.CreateRet(gf_retval); break; } @@ -4107,7 +3675,8 @@ static void emit_cfunc_invalidate( } static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_tupletype_t *argt, - jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt) + jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt, + jl_codegen_params_t ¶ms) { // Generate a c-callable wrapper bool toboxed; @@ -4129,18 +3698,23 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // try to look up this function for direct invoking jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world); jl_value_t *astrt = (jl_value_t*)jl_any_type; + jl_llvm_functions_t decls = {}; // infer it first, if necessary if (lam) { name = jl_symbol_name(lam->def.method->name); - jl_code_info_t *src = NULL; - if (!lam->inferred) // TODO: this isn't ideal to be unconditionally calling type inference from here - src = jl_type_infer(&lam, world, 0); - jl_compile_linfo(&lam, src, world, &jl_default_cgparams); - if (lam->jlcall_api != JL_API_CONST) { - if (lam->functionObjectsDecls.functionObject == NULL || - jl_jlcall_api(lam->functionObjectsDecls.functionObject) != 1) { - lam = NULL; // TODO: use emit_invoke framework to dispatch these - } + // TODO: this isn't ideal to be causing recursion from here + jl_generic_fptr_t fptr = jl_generate_fptr(&lam, world); + if (lam->jlcall_api == JL_API_CONST) { + // don't need the fptr + } + else if (fptr.jlcall_api == JL_API_GENERIC) { + assert(fptr.fptr); + decls.functionObject = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)fptr.fptr, lam); + if (lam->fptr_specsig) + decls.specFunctionObject = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)lam->fptr_specsig, lam); + } + else { + lam = NULL; // TODO: use emit_invoke framework to dispatch these } if (lam) { astrt = lam->rettype; @@ -4163,18 +3737,16 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t Function *cw = Function::Create(sig.functype, GlobalVariable::ExternalLinkage, funcName.str(), M); - jl_init_function(cw); cw->setAttributes(sig.attributes); #ifdef JL_DISABLE_FPO cw->addFnAttr("no-frame-pointer-elim", "true"); #endif Function *cw_proto = function_proto(cw); - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = cw; ctx.linfo = lam; ctx.world = world; - ctx.params = &jl_default_cgparams; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", cw); ctx.builder.SetInsertPoint(b0); @@ -4295,9 +3867,9 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t jlfunc_sret = false; retval = mark_julia_const(lam->inferred_const); } - else if (lam && lam->functionObjectsDecls.specFunctionObject != NULL) { + else if (lam && !decls.specFunctionObject.empty()) { // emit a specsig call - const char *protoname = lam->functionObjectsDecls.specFunctionObject; + StringRef protoname = decls.specFunctionObject; jl_returninfo_t returninfo = get_specsig_function(M, protoname, lam->specTypes, lam->rettype); FunctionType *cft = returninfo.decl->getFunctionType(); jlfunc_sret = (returninfo.cc == jl_returninfo_t::SRet); @@ -4344,7 +3916,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t funcName << "_gfthunk"; Function *gf_thunk = Function::Create(returninfo.decl->getFunctionType(), GlobalVariable::InternalLinkage, funcName.str(), M); - jl_init_function(gf_thunk); gf_thunk->setAttributes(returninfo.decl->getAttributes()); #ifdef JL_DISABLE_FPO gf_thunk->addFnAttr("no-frame-pointer-elim", "true"); @@ -4352,7 +3923,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // build a specsig -> jl_apply_generic converter thunk // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), // but which has the signature of a specsig - emit_cfunc_invalidate(gf_thunk, returninfo.cc, lam, nargs + 1, world); + emit_cfunc_invalidate(gf_thunk, returninfo.cc, lam->specTypes, lam->rettype, nargs + 1, ctx.emission_context); theFptr = ctx.builder.CreateSelect(age_ok, theFptr, gf_thunk); } CallInst *call = ctx.builder.CreateCall(theFptr, ArrayRef(args)); @@ -4384,8 +3955,8 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t jlfunc_sret = false; Function *theFptr = NULL; if (lam) { - const char *fname = lam->functionObjectsDecls.functionObject; - if (fname) { + if (!decls.functionObject.empty()) { + StringRef fname = decls.functionObject; theFptr = cast_or_null(jl_Module->getNamedValue(fname)); if (!theFptr) { theFptr = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, @@ -4465,7 +4036,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t ctx.builder.SetCurrentDebugLocation(noDbg); ctx.builder.ClearInsertionPoint(); - jl_finalize_module(M, true); + jl_finalize_module(std::unique_ptr(M)); return cw_proto; } @@ -4477,7 +4048,8 @@ const struct jl_typemap_info cfunction_cache = { // Get the LLVM Function* for the C-callable entry point for a certain function // and argument types. // here argt does not include the leading function type argument -static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_tupletype_t *argt) +Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_tupletype_t *argt, + jl_codegen_params_t ¶ms) { // Assumes the codegen lock is acquired. The caller is responsible for that. @@ -4550,31 +4122,19 @@ static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_t } // Backup the info for the nested compile - bool last_n_c = nested_compile; - nested_compile = true; - Function *f; - JL_TRY { - f = gen_cfun_wrapper(ff, crt, (jl_tupletype_t*)argt, sf, declrt, (jl_tupletype_t*)sigt); - } - JL_CATCH { - f = NULL; - } - // Restore the previous compile context - nested_compile = last_n_c; + Function *f = gen_cfun_wrapper(ff, crt, (jl_tupletype_t*)argt, sf, declrt, (jl_tupletype_t*)sigt, params); JL_GC_POP(); - if (f == NULL) - jl_rethrow(); return f; } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returninfo_t &f, const std::string &funcName, Module *M) +static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returninfo_t &f, StringRef funcName, + Module *M, jl_codegen_params_t ¶ms) { Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, funcName, M); add_return_attr(w, Attribute::NonNull); w->addFnAttr("thunk"); - jl_init_function(w); #ifdef JL_DISABLE_FPO w->addFnAttr("no-frame-pointer-elim", "true"); #endif @@ -4583,11 +4143,10 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returnin Value *argArray = &*AI++; /* const Argument &argCount = *AI++; */ - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = w; ctx.linfo = lam; ctx.world = 0; - ctx.params = &jl_default_cgparams; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", w); ctx.builder.SetInsertPoint(b0); @@ -4674,50 +4233,7 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returnin return w; } -static bool uses_specsig(jl_value_t *sig, size_t nreq, jl_value_t *rettype, bool needsparam, bool va, jl_code_info_t *src, bool prefer_specsig) -{ - if (needsparam) - return false; - if (!src || !jl_ast_flag_inferred((jl_array_t*)src)) - return false; - if (sig == (jl_value_t*)jl_anytuple_type) - return false; - if (!jl_is_datatype(sig)) - return false; - if (jl_nparams(sig) == 0) - return false; - if (va) { - if (jl_is_vararg_type(jl_tparam(sig, jl_nparams(sig)-1))) - return false; - // For now we can only handle va tuples that will end up being - // leaf types - for (size_t i = nreq; i < jl_nparams(sig); i++) { - if (!jl_justbits(jl_tparam(sig, i))) - return false; - } - } - // not invalid, consider if specialized signature is worthwhile - if (prefer_specsig) - return true; - if (jl_justbits(rettype) && !jl_is_datatype_singleton((jl_datatype_t*)rettype)) - return true; - if (jl_is_uniontype(rettype)) { - bool allunbox; - size_t nbytes, align, min_align; - union_alloca_type((jl_uniontype_t*)rettype, allunbox, nbytes, align, min_align); - if (nbytes > 0) - return true; // some elements of the union could be returned unboxed avoiding allocation - } - for (size_t i = 0; i < jl_nparams(sig); i++) { - jl_value_t *sigt = jl_tparam(sig, i); - if (jl_justbits(sigt) && !jl_is_datatype_singleton((jl_datatype_t*)sigt)) { - return true; - } - } - return false; // jlcall sig won't require any box allocations -} - -static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype) +static jl_returninfo_t get_specsig_function(Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype) { jl_returninfo_t props = {}; SmallVector fsig; @@ -4838,19 +4354,17 @@ static jl_datatype_t *compute_va_type(jl_method_instance_t *lam, size_t nreq) } // Compile to LLVM IR, using a specialized signature if applicable. -static std::unique_ptr emit_function( +static std::pair, jl_llvm_functions_t> + emit_function( jl_method_instance_t *lam, jl_code_info_t *src, - size_t world, - jl_llvm_functions_t *declarations, - const jl_cgparams_t *params) + jl_codegen_params_t ¶ms) { - assert(declarations && "Capturing declarations is always required"); - // step 1. unpack AST and allocate codegen context for this function - jl_codectx_t ctx(jl_LLVMContext); + jl_llvm_functions_t declarations; + jl_codectx_t ctx(jl_LLVMContext, params); JL_GC_PUSH2(&ctx.code, &ctx.roots); - ctx.code = (jl_array_t*)src->code; + ctx.code = src->code; //jl_static_show(JL_STDOUT, (jl_value_t*)ast); //jl_printf(JL_STDOUT, "\n"); @@ -4860,10 +4374,8 @@ static std::unique_ptr emit_function( ctx.module = jl_is_method(lam->def.method) ? lam->def.method->module : lam->def.module; ctx.linfo = lam; ctx.source = src; - ctx.world = world; ctx.name = name_from_method_instance(lam); ctx.funcName = ctx.name; - ctx.params = params; ctx.spvals_ptr = NULL; ctx.nargs = jl_is_method(lam->def.method) ? lam->def.method->nargs : 0; bool toplevel = !jl_is_method(lam->def.method); @@ -4924,18 +4436,11 @@ static std::unique_ptr emit_function( ctx.ssavalue_assigned.assign(n_ssavalues, false); ctx.SAvalues.assign(n_ssavalues, jl_cgval_t()); - bool needsparams = false; - if (jl_is_method(lam->def.method)) { - if (jl_svec_len(lam->def.method->sparam_syms) != jl_svec_len(lam->sparam_vals)) - needsparams = true; - for (size_t i = 0; i < jl_svec_len(lam->sparam_vals); ++i) { - if (jl_is_typevar(jl_svecref(lam->sparam_vals, i))) - needsparams = true; - } - } - jl_value_t *jlrettype = lam->rettype; - bool specsig = uses_specsig(lam->specTypes, nreq, jlrettype, needsparams, va, src, params->prefer_specsig); + bool specsig, needsparams; + std::tie(specsig, needsparams) = uses_specsig(lam, params.params->prefer_specsig); + if (!src->inferred) + specsig = false; // step 3. some variable analysis size_t i; @@ -4993,35 +4498,29 @@ static std::unique_ptr emit_function( unadorned_name++; #endif funcName << unadorned_name << "_" << globalUnique++; + declarations.functionObject = funcName.str(); // allocate Function declarations and wrapper objects Module *M = new Module(ctx.name, jl_LLVMContext); - jl_setup_module(M, params); + jl_setup_module(M, ctx.params); jl_returninfo_t returninfo = {}; - Function *f = NULL; - Function *fwrap = NULL; + Function *f; if (specsig) { // assumes !va and !needsparams std::stringstream specName; specName << "julia_" << unadorned_name << "_" << globalUnique; - returninfo = get_specsig_function(M, specName.str(), lam->specTypes, jlrettype); + declarations.specFunctionObject = specName.str(); + returninfo = get_specsig_function(M, declarations.specFunctionObject, lam->specTypes, jlrettype); f = returninfo.decl; ctx.has_sret = (returninfo.cc == jl_returninfo_t::SRet || returninfo.cc == jl_returninfo_t::Union); - jl_init_function(f); - - fwrap = gen_jlcall_wrapper(lam, returninfo, funcName.str(), M); - declarations->functionObject = strdup(fwrap->getName().str().c_str()); - declarations->specFunctionObject = strdup(f->getName().str().c_str()); + gen_jlcall_wrapper(lam, returninfo, declarations.functionObject, M, ctx.emission_context); } else { f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, GlobalVariable::ExternalLinkage, - funcName.str(), M); + declarations.functionObject, M); add_return_attr(f, Attribute::NonNull); f->addFnAttr("thunk"); returninfo.decl = f; - jl_init_function(f); - declarations->functionObject = strdup(f->getName().str().c_str()); - declarations->specFunctionObject = NULL; } #ifdef JL_DISABLE_FPO @@ -5904,9 +5403,179 @@ static std::unique_ptr emit_function( } JL_GC_POP(); - return std::unique_ptr(M); + return std::make_pair(std::unique_ptr(M), declarations); } +void jl_add_linfo_in_flight(StringRef name, jl_method_instance_t *linfo, const DataLayout &DL); + +static int32_t jl_jlcall_api(StringRef fname) +{ + // TODO: it would really make more sense to just return this explicitly + if (fname.empty()) + return JL_API_NOT_SET; + if (fname.startswith("japi3_")) // jlcall abi 3 from JIT + return JL_API_WITH_PARAMETERS; + assert(fname.startswith("japi1_") || // jlcall abi 1 from JIT + fname.startswith("jlcall_")); // jlcall abi 1 from JIT wrapping a specsig method + return JL_API_GENERIC; +} + +jl_compile_result_t jl_compile_linfo1( + jl_method_instance_t *li, + jl_code_info_t *src, + jl_codegen_params_t ¶ms) +{ + // caller must hold codegen_lock + jl_llvm_functions_t decls = {}; + std::unique_ptr m; + assert(li->min_world <= params.world && (li->max_world >= params.world || li->max_world == 0) && + "invalid world for method-instance"); + assert((params.params == &jl_default_cgparams /* fast path */ || !params.cache || + compare_cgparams(params.params, &jl_default_cgparams)) && + "functions compiled with custom codegen params must not be cached"); + JL_TRY { + std::tie(m, decls) = emit_function(li, src, params); + } + JL_CATCH { + // Something failed! This is very, very bad. + // Try to pretend that it isn't and attempt to recover. + jl_ptls_t ptls = jl_get_ptls_states(); + m.reset(); + decls.functionObject = ""; + decls.specFunctionObject = ""; + const char *mname = name_from_method_instance(li); + jl_printf(JL_STDERR, "Internal error: encountered unexpected error during compilation of %s:\n", mname); + jl_static_show(JL_STDERR, ptls->exception_in_transit); + jl_printf(JL_STDERR, "\n"); + ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE); // record_backtrace for some context + jlbacktrace(); // written to STDERR_FILENO + } + + if (params.cache && !decls.functionObject.empty()) { + // Prepare debug info to receive this function + // record that this function name came from this linfo, + // so we can build a reverse mapping for debug-info. + if (!JL_HOOK_TEST(params.params, module_activation)) { + bool toplevel = !jl_is_method(li->def.method); + if (!toplevel) { + const DataLayout &DL = m->getDataLayout(); + // but don't remember toplevel thunks because + // they may not be rooted in the gc for the life of the program, + // and the runtime doesn't notify us when the code becomes unreachable :( + const std::string &f = (decls.specFunctionObject.empty() ? decls.functionObject : decls.specFunctionObject); + jl_add_linfo_in_flight(StringRef(f), li, DL); + } + } + + // if not inlineable, code won't be needed again + if (JL_DELETE_NON_INLINEABLE && + // don't delete code when debugging level >= 2 + jl_options.debug_level <= 1 && + // don't delete toplevel code + jl_is_method(li->def.method) && + // don't change inferred state + li->inferred && + // and there is something to delete (test this before calling jl_ast_flag_inlineable) + li->inferred != jl_nothing && + // don't delete inlineable code, unless it is constant + (li->jlcall_api == JL_API_CONST || !jl_ast_flag_inlineable((jl_array_t*)li->inferred)) && + // don't delete code when generating a precompile file + !imaging_mode && + // don't delete code when it's not actually directly being used + params.world) { + li->inferred = jl_nothing; + } + } + + return std::make_tuple(std::move(m), decls, li->rettype, jl_jlcall_api(decls.functionObject)); +} + +void jl_compile_workqueue( + std::map &emitted, + jl_codegen_params_t ¶ms) +{ + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + while (!params.workqueue.empty()) { + jl_llvm_functions_t decls = {}; + jl_method_instance_t *li; + Function *protodecl; + jl_returninfo_t::CallingConv proto_cc; + bool proto_specsig; + jl_value_t *proto_rettype; + std::tie(li, proto_cc, protodecl, proto_rettype, proto_specsig) = params.workqueue.back(); + params.workqueue.pop_back(); + // try to emit code for this item from the workqueue + jl_value_t *rettype = li->rettype; + if (li->min_world > params.world || li->max_world < params.world) { + // assert(0); + } + else if (params.cache && li->fptr && li->jlcall_api == JL_API_GENERIC) { + + decls.functionObject = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)li->fptr, li); + if (li->fptr_specsig) + decls.specFunctionObject = jl_ExecutionEngine->getFunctionAtAddress((uint64_t)(uintptr_t)li->fptr_specsig, li); + } + else { + jl_compile_result_t &result = emitted[li]; + if (std::get<0>(result)) { + decls = std::get<1>(result); + rettype = std::get<2>(result); + } + else { + src = (jl_code_info_t*)li->inferred; + if (src && (jl_value_t*)src != jl_nothing) { + if (jl_is_method(li->def.method)) + src = jl_uncompress_ast(li->def.method, (jl_array_t*)src); + if (jl_is_code_info(src)) { + result = jl_compile_linfo1(li, src, params); + } + } + if (std::get<0>(result)) { + decls = std::get<1>(result); + rettype = std::get<2>(result); + } + else { + emitted.erase(li); + } + } + } + // patch up the prototype we emitted earlier + Module *mod = protodecl->getParent(); + StringRef preal_decl = ""; + assert(protodecl->isDeclaration()); + if (proto_specsig) { + preal_decl = decls.specFunctionObject; + if (!jl_egal(proto_rettype, rettype) || preal_decl.empty()) { + protodecl->setLinkage(GlobalVariable::PrivateLinkage); + protodecl->addFnAttr("no-frame-pointer-elim", "true"); + size_t nargs = jl_nparams(li->specTypes); // number of actual arguments being passed + // TODO: might be nice to manage to cache this + emit_cfunc_invalidate(protodecl, proto_cc, li->specTypes, proto_rettype, nargs, params); + preal_decl = ""; + } + } + else { + preal_decl = decls.functionObject; + if (preal_decl.empty()) { + // TODO: generate an indirect call to li->fptr1 trampoline + preal_decl = "jl_apply_2va"; + } + } + if (!preal_decl.empty()) { + if (Value *specfun = mod->getNamedValue(preal_decl)) { + if (protodecl != specfun) + protodecl->replaceAllUsesWith(specfun); + } + else { + protodecl->setName(preal_decl); + } + } + } + JL_GC_POP(); +} + + // --- initialization --- std::pair tbaa_make_child(const char *name, MDNode *parent=nullptr, bool isConstant=false) @@ -5953,43 +5622,6 @@ static Function *jlcall_func_to_llvm(const std::string &cname, jl_fptr_t addr, M return f; } -extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_method_instance_t *lam, int specsig) -{ - if (imaging_mode) { - if (!specsig) { - lam->fptr = fptr; // in imaging mode, it's fine to use the fptr, but we don't want it in the shadow_module - } - } - else { - // this assigns a function pointer (from loading the system image), to the function object - std::stringstream funcName; - if (specsig) - funcName << "jlsys_"; // the specsig implementation - else if (lam->functionObjectsDecls.specFunctionObject) - funcName << "jlsysw_"; // it's a specsig wrapper - else if (lam->jlcall_api == JL_API_GENERIC) - funcName << "jsys1_"; // it's a jlcall without a specsig - const char* unadorned_name = jl_symbol_name(lam->def.method->name); - funcName << unadorned_name << "_" << globalUnique++; - Function *f = jlcall_func_to_llvm(funcName.str(), fptr, NULL); - if (specsig) { - if (lam->functionObjectsDecls.specFunctionObject == NULL) { - lam->functionObjectsDecls.specFunctionObject = strdup(f->getName().str().c_str()); - } - } - else { - assert(lam->fptr == NULL); - lam->fptr = fptr; - if (lam->jlcall_api == JL_API_GENERIC) { - if (lam->functionObjectsDecls.functionObject == NULL) { - lam->functionObjectsDecls.functionObject = strdup(f->getName().str().c_str()); - } - } - } - delete f; - } -} - static void init_julia_llvm_meta(void) { tbaa_gcframe = tbaa_make_child("jtbaa_gcframe").first; @@ -6101,7 +5733,6 @@ static void init_julia_llvm_env(Module *m) four_pvalue_llvmt.push_back(T_pjlvalue); four_pvalue_llvmt.push_back(T_pjlvalue); V_null = Constant::getNullValue(T_pjlvalue); - jl_init_jit(T_pjlvalue); std::vector ftargs(0); ftargs.push_back(T_pprjlvalue); // linfo->sparam_vals @@ -6615,13 +6246,9 @@ static void init_julia_llvm_env(Module *m) false, GlobalVariable::ExternalLinkage, NULL, "jl_world_counter"); add_named_global(jlgetworld_global, &jl_world_counter); - - jl_globalPM = new legacy::PassManager(); - addTargetPasses(jl_globalPM, jl_TargetMachine); - addOptimizationPasses(jl_globalPM, jl_options.opt_level); } -extern "C" void *jl_init_llvm(void) +extern "C" void jl_init_llvm(void) { const char *const argv_tailmerge[] = {"", "-enable-tail-merge=0"}; // NOO TOUCHIE; NO TOUCH! See #922 cl::ParseCommandLineOptions(sizeof(argv_tailmerge)/sizeof(argv_tailmerge[0]), argv_tailmerge, "disable-tail-merge\n"); @@ -6648,11 +6275,6 @@ extern "C" void *jl_init_llvm(void) InitializeNativeTargetAsmParser(); InitializeNativeTargetDisassembler(); - Module *m, *engine_module; - engine_module = new Module("julia", jl_LLVMContext); - m = new Module("julia", jl_LLVMContext); - shadow_output = m; - TargetOptions options = TargetOptions(); //options.PrintMachineCode = true; //Print machine code produced during JIT compiling #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) @@ -6661,25 +6283,6 @@ extern "C" void *jl_init_llvm(void) // to ensure compatibility with GCC codes options.StackAlignmentOverride = 16; #endif - EngineBuilder eb((std::unique_ptr(engine_module))); - std::string ErrorStr; - eb .setEngineKind(EngineKind::JIT) - .setTargetOptions(options) - // Generate simpler code for JIT - .setRelocationModel(Reloc::Static) -#ifdef _P64 - // Make sure we are using the large code model on 64bit - // Let LLVM pick a default suitable for jitting on 32bit - .setCodeModel(CodeModel::Large) -#elif JL_LLVM_VERSION < 60000 - .setCodeModel(CodeModel::JITDefault) -#endif -#ifdef DISABLE_OPT - .setOptLevel(CodeGenOpt::None) -#else - .setOptLevel(jl_options.opt_level == 0 ? CodeGenOpt::None : CodeGenOpt::Aggressive) -#endif - ; Triple TheTriple(sys::getProcessTriple()); #if defined(FORCE_ELF) TheTriple.setObjectFormat(Triple::ELF); @@ -6688,12 +6291,13 @@ extern "C" void *jl_init_llvm(void) auto target = jl_get_llvm_target(imaging_mode, target_flags); auto &TheCPU = target.first; SmallVector targetFeatures(target.second.begin(), target.second.end()); + std::string errorstr; + const Target *TheTarget = TargetRegistry::lookupTarget("", TheTriple, errorstr); + if (!TheTarget) + jl_errorf("%s", errorstr.c_str()); if (jl_processor_print_help || (target_flags & JL_TARGET_UNKNOWN_NAME)) { - std::string errorstr; - const Target *target = TargetRegistry::lookupTarget("", TheTriple, errorstr); - assert(target); std::unique_ptr MSTI( - target->createMCSubtargetInfo(TheTriple.str(), "", "")); + TheTarget->createMCSubtargetInfo(TheTriple.str(), "", "")); if (!MSTI->isCPUStringValid(TheCPU)) jl_errorf("Invalid CPU name %s.", TheCPU.c_str()); if (jl_processor_print_help) { @@ -6703,11 +6307,41 @@ extern "C" void *jl_init_llvm(void) MSTI->setDefaultFeatures("help", ""); } } - jl_TargetMachine = eb.selectTarget( - TheTriple, - "", - TheCPU, - targetFeatures); + // Package up features to be passed to target/subtarget + std::string FeaturesStr; + if (!targetFeatures.empty()) { + SubtargetFeatures Features; + for (unsigned i = 0; i != targetFeatures.size(); ++i) + Features.AddFeature(targetFeatures[i]); + FeaturesStr = Features.getString(); + } + // Allocate a target... + auto codemodel = +#ifdef _P64 + // Make sure we are using the large code model on 64bit + // Let LLVM pick a default suitable for jitting on 32bit + CodeModel::Large; +#elif JL_LLVM_VERSION < 60000 + CodeModel::JITDefault; +#else + CodeModel::Default; +#endif + auto optlevel = +#ifdef DISABLE_OPT + CodeGenOpt::None; +#else + (jl_options.opt_level == 0 ? CodeGenOpt::None : CodeGenOpt::Aggressive); +#endif + jl_TargetMachine = TheTarget->createTargetMachine( + TheTriple.getTriple(), TheCPU, FeaturesStr, + options, + Reloc::Static, // Generate simpler code for JIT + codemodel, + optlevel +#if JL_LLVM_VERSION >= 60000 + , /*JIT*/ true +#endif + ); assert(jl_TargetMachine && "Failed to select target machine -" " Is the LLVM backend for this CPU enabled?"); #if (!defined(_CPU_ARM_) && !defined(_CPU_PPC64_)) @@ -6725,16 +6359,16 @@ extern "C" void *jl_init_llvm(void) std::string DL = jl_data_layout.getStringRepresentation() + "-ni:10:11:12"; jl_data_layout.reset(DL); #endif - - // Now that the execution engine exists, initialize all modules - jl_setup_module(engine_module); - jl_setup_module(m); - return (void*)m; } extern "C" void jl_init_codegen(void) { - Module *m = (Module *)jl_init_llvm(); + jl_init_llvm(); + // Now that the execution engine exists, initialize all modules + jl_init_jit(); + Module *m = new Module("julia", jl_LLVMContext); + shadow_output = m; + jl_setup_module(m); init_julia_llvm_env(m); BOX_F(int8,int8); UBOX_F(uint8,uint8); @@ -6754,9 +6388,14 @@ extern "C" void jl_init_codegen(void) box64_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int64), "jl_box64", &jl_box64, m); jl_init_intrinsic_functions_codegen(m); + + jl_globalPM = new legacy::PassManager(); + addTargetPasses(jl_globalPM, jl_TargetMachine); + addOptimizationPasses(jl_globalPM, jl_options.opt_level); } -// for debugging from gdb +// the rest of this file are convenience functions +// that are exported for assisting with debugging from gdb extern "C" void jl_dump_llvm_value(void *v) { llvm_dump((Value*)v); diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 970b50e595494..3c2ca9978f49b 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -159,19 +159,6 @@ struct strrefcomp { } }; -extern "C" tracer_cb jl_linfo_tracer; -static std::vector triggered_linfos; -void jl_callback_triggered_linfos(void) -{ - if (triggered_linfos.empty()) - return; - if (jl_linfo_tracer) { - std::vector to_process(std::move(triggered_linfos)); - for (jl_method_instance_t *linfo : to_process) - jl_call_tracer(jl_linfo_tracer, (jl_value_t*)linfo); - } -} - class JuliaJITEventListener: public JITEventListener { std::map objectmap; @@ -378,20 +365,7 @@ class JuliaJITEventListener: public JITEventListener jl_method_instance_t *linfo = NULL; if (linfo_it != linfo_in_flight.end()) { linfo = linfo_it->second; - if (linfo->compile_traced) - triggered_linfos.push_back(linfo); linfo_in_flight.erase(linfo_it); - const char *F = linfo->functionObjectsDecls.functionObject; - if (!linfo->fptr && F && sName.equals(F)) { - int jlcall_api = jl_jlcall_api(F); - if (linfo->inferred || jlcall_api != JL_API_GENERIC) { - linfo->jlcall_api = jlcall_api; - linfo->fptr = (jl_fptr_t)(uintptr_t)Addr; - } - else { - linfo->unspecialized_ducttape = (jl_fptr_t)(uintptr_t)Addr; - } - } } if (linfo) linfomap[Addr] = std::make_pair(Size, linfo); diff --git a/src/disasm.cpp b/src/disasm.cpp index 3aea6151737db..9ce938acfb477 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -259,6 +259,62 @@ void LineNumberAnnotatedWriter::emitBasicBlockEndAnnot( LinePrinter.emit_finish(Out); } +static void jl_strip_llvm_debug(Module *m, bool all_meta, LineNumberAnnotatedWriter *AAW) +{ + // strip metadata from all instructions in all functions in the module + Instruction *deletelast = nullptr; // can't actually delete until the iterator advances + for (Function &f : m->functions()) { + if (AAW) + AAW->addSubprogram(&f, f.getSubprogram()); + for (BasicBlock &f_bb : f) { + for (Instruction &inst : f_bb) { + if (deletelast) { + deletelast->eraseFromParent(); + deletelast = nullptr; + } + // remove dbg.declare and dbg.value calls + if (isa(inst) || isa(inst)) { + deletelast = &inst; + continue; + } + + // iterate over all metadata kinds and set to NULL to remove + if (all_meta) { + SmallVector, 4> MDForInst; + inst.getAllMetadataOtherThanDebugLoc(MDForInst); + for (const auto &md_iter : MDForInst) { + inst.setMetadata(md_iter.first, NULL); + } + } + // record debug location before erasing it + if (AAW) + AAW->addDebugLoc(&inst, inst.getDebugLoc()); + inst.setDebugLoc(DebugLoc()); + } + if (deletelast) { + deletelast->eraseFromParent(); + deletelast = nullptr; + } + } + f.setSubprogram(NULL); + } + if (all_meta) { + for (GlobalObject &g : m->global_objects()) { + g.clearMetadata(); + } + } + // now that the subprogram is not referenced, we can delete it too + if (NamedMDNode *md = m->getNamedMetadata("llvm.dbg.cu")) + m->eraseNamedMetadata(md); + //if (NamedMDNode *md = m->getNamedMetadata("llvm.module.flags")) + // m->eraseNamedMetadata(md); +} + +void jl_strip_llvm_debug(Module *m) +{ + jl_strip_llvm_debug(m, false, NULL); +} + // print an llvm IR acquired from jl_get_llvmf // warning: this takes ownership of, and destroys, f->getParent() extern "C" JL_DLLEXPORT @@ -280,43 +336,8 @@ jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_modul } else { Module *m = llvmf->getParent(); - if (strip_ir_metadata) { - // strip metadata from all instructions in all functions in the module - Instruction *deletelast = nullptr; // can't actually delete until the iterator advances - for (Function &f2 : m->functions()) { - AAW.addSubprogram(&f2, f2.getSubprogram()); - for (BasicBlock &f2_bb : f2) { - for (Instruction &inst : f2_bb) { - if (deletelast) { - deletelast->eraseFromParent(); - deletelast = nullptr; - } - // remove dbg.declare and dbg.value calls - if (isa(inst) || isa(inst)) { - deletelast = &inst; - continue; - } - - // iterate over all metadata kinds and set to NULL to remove - SmallVector, 4> MDForInst; - inst.getAllMetadataOtherThanDebugLoc(MDForInst); - for (const auto &md_iter : MDForInst) { - inst.setMetadata(md_iter.first, NULL); - } - // record debug location before erasing it - AAW.addDebugLoc(&inst, inst.getDebugLoc()); - inst.setDebugLoc(DebugLoc()); - } - if (deletelast) { - deletelast->eraseFromParent(); - deletelast = nullptr; - } - } - } - for (GlobalObject &g2 : m->global_objects()) { - g2.clearMetadata(); - } - } + if (strip_ir_metadata) + jl_strip_llvm_debug(m, true, &AAW); if (dump_module) { m->print(stream, &AAW); } @@ -528,8 +549,9 @@ void SymbolTable::createSymbols() } else { const char *global = lookupLocalPC(addr); - if (global) + if (global || !global[0]) isymb->second = global; + // TODO: free(global)? } } } diff --git a/src/dump.c b/src/dump.c index 71416cfb52de8..eee23f0f868a8 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1538,7 +1538,6 @@ static jl_value_t *jl_deserialize_value_method_instance(jl_serializer_state *s, li->backedges = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&li->backedges); if (li->backedges) jl_gc_wb(li, li->backedges); - li->unspecialized_ducttape = NULL; if (internal == 1) { li->min_world = 0; li->max_world = 0; @@ -1555,11 +1554,15 @@ static jl_value_t *jl_deserialize_value_method_instance(jl_serializer_state *s, assert(0 && "corrupt deserialization state"); abort(); } - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; li->inInference = 0; li->fptr = NULL; + li->fptr_specsig = NULL; li->jlcall_api = read_int8(s->s); + if (li->jlcall_api == JL_API_CONST) { + li->fptr = (jl_fptr_t)(void*)li->inferred_const; + if (!li->fptr) // corrupt? + li->jlcall_api = 0; + } li->compile_traced = 0; return (jl_value_t*)li; } diff --git a/src/gf.c b/src/gf.c index 62806ce569820..f5f69bf9e07ce 100644 --- a/src/gf.c +++ b/src/gf.c @@ -243,6 +243,8 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t **pli, size_t world, int forc jl_method_instance_t *li = *pli; if (li->inInference && !force) return NULL; + if (jl_is_method(li->def.method) && li->def.method->unspecialized == li) + return NULL; // be careful never to infer the unspecialized method, this would not be valid jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 3); @@ -279,11 +281,12 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t **pli, size_t world, int forc int jl_is_rettype_inferred(jl_method_instance_t *li) { - if (!li->inferred) + jl_value_t *code = li->inferred; + if (!code) return 0; - if (jl_is_code_info(li->inferred) && !((jl_code_info_t*)li->inferred)->inferred) - return 0; - return 1; + if (code == jl_nothing || jl_ast_flag_inferred((jl_array_t*)code)) + return 1; + return 0; } struct set_world { @@ -404,14 +407,14 @@ JL_DLLEXPORT jl_method_instance_t* jl_set_method_inferred( // changing rettype changes the llvm signature, // so clear all of the llvm state at the same time - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; + li->fptr_specsig = NULL; li->rettype = rettype; jl_gc_wb(li, rettype); li->inferred = inferred; jl_gc_wb(li, inferred); if (const_flags & 1) { assert(const_flags & 2); + li->fptr = (jl_fptr_t)(void*)inferred_const; li->jlcall_api = JL_API_CONST; } if (const_flags & 2) { @@ -1725,66 +1728,79 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int return ml_matches(mt->defs, 0, types, lim, include_ambiguous, world, min_valid, max_valid); } -jl_llvm_functions_t jl_compile_for_dispatch(jl_method_instance_t **pli, size_t world) +static jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method) +{ + // one unspecialized version of a function can be shared among all cached specializations + jl_method_t *def = method->def.method; + if (def->source == NULL) { + // generated functions might instead randomly just never get inferred, sorry + return method; + } + if (def->unspecialized == NULL) { + JL_LOCK(&def->writelock); + if (def->unspecialized == NULL) { + def->unspecialized = jl_get_specialized(def, def->sig, jl_emptysvec); + jl_gc_wb(def, def->unspecialized); + } + JL_UNLOCK(&def->writelock); + } + return def->unspecialized; +} + +jl_generic_fptr_t jl_compile_for_dispatch(jl_method_instance_t **pli, size_t world) { + jl_generic_fptr_t fptr; jl_method_instance_t *li = *pli; - if (li->jlcall_api == JL_API_CONST) - return li->functionObjectsDecls; + if (li->jlcall_api != JL_API_NOT_SET) { + fptr.jlcall_api = li->jlcall_api; + fptr.fptr = li->fptr; + return fptr; + } if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) { // copy fptr from the template method definition jl_method_t *def = li->def.method; if (jl_is_method(def) && def->unspecialized) { - if (def->unspecialized->jlcall_api == JL_API_CONST) { - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; - li->inferred = def->unspecialized->inferred; - jl_gc_wb(li, li->inferred); - li->inferred_const = def->unspecialized->inferred_const; - if (li->inferred_const) - jl_gc_wb(li, li->inferred_const); - li->jlcall_api = JL_API_CONST; - return li->functionObjectsDecls; - } if (def->unspecialized->fptr) { - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; + if (def->unspecialized->jlcall_api == JL_API_CONST) + li->inferred_const = def->unspecialized->inferred_const; li->jlcall_api = def->unspecialized->jlcall_api; li->fptr = def->unspecialized->fptr; - return li->functionObjectsDecls; + fptr.jlcall_api = li->jlcall_api; + fptr.fptr = li->fptr; + return fptr; } } if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF && - jl_is_method(li->def.method)) { + jl_is_method(li->def.method)) { jl_code_info_t *src = jl_code_for_interpreter(li); if (!jl_code_requires_compiler(src)) { li->inferred = (jl_value_t*)src; jl_gc_wb(li, src); - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; li->fptr = (jl_fptr_t)&jl_interpret_call; li->jlcall_api = JL_API_INTERPRETED; - return li->functionObjectsDecls; + fptr.jlcall_api = JL_API_INTERPRETED; + fptr.fptr = li->fptr; + return fptr; } } } - jl_llvm_functions_t decls = li->functionObjectsDecls; - if (decls.functionObject != NULL || li->jlcall_api == JL_API_CONST) - return decls; - jl_code_info_t *src = NULL; - if (jl_is_method(li->def.method) && !jl_is_rettype_inferred(li) && - jl_symbol_name(li->def.method->name)[0] != '@') { - // don't bother with typeinf on macros or toplevel thunks - // but try to infer everything else - src = jl_type_infer(pli, world, 0); - li = *pli; + fptr = jl_generate_fptr(pli, world); + if (fptr.jlcall_api == JL_API_NOT_SET && jl_is_method(li->def.method)) { + // couldn't compile F in the JIT right now, + // so instead get an unspecialized (uninferred) version of the code + // and return its fptr instead + // TODO: always just run this in the interpreter instead? + fptr = jl_generate_fptr_for_unspecialized(jl_get_unspecialized(li)); + } + if (fptr.jlcall_api == JL_API_NOT_SET) { + // this should pretty much never happen, but will if the compiler fails + // (because of staged functions or llvmcall or other codegen bugs, for example) + fptr.fptr = (jl_fptr_t)&jl_interpret_call; + fptr.jlcall_api = JL_API_INTERPRETED; } - // check again, because jl_type_infer may have changed li or compiled it - decls = li->functionObjectsDecls; - if (decls.functionObject != NULL || li->jlcall_api == JL_API_CONST) - return decls; - return jl_compile_linfo(&li, src, world, &jl_default_cgparams); + return fptr; } // compile-time method lookup @@ -1845,14 +1861,11 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) jl_method_instance_t *li = jl_get_specialization1(types, world); if (li == NULL) return 0; - jl_code_info_t *src = NULL; - if (!jl_is_rettype_inferred(li)) - src = jl_type_infer(&li, world, 0); if (li->jlcall_api != JL_API_CONST) { if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc) { // If we are saving LLVM or native code, generate the LLVM IR so that it'll // be included in the saved LLVM module. - jl_compile_linfo(&li, src, world, &jl_default_cgparams); + (void)jl_generate_fptr(&li, world); } else if (!jl_options.outputji) { // If we are only saving ji files (e.g. package pre-compilation for now), @@ -1862,10 +1875,12 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) jl_ptls_t ptls = jl_get_ptls_states(); size_t last_age = ptls->world_age; ptls->world_age = world; - jl_generic_fptr_t fptr; - jl_compile_method_internal(&fptr, li); + (void)jl_compile_method_internal(li); ptls->world_age = last_age; } + else if (!jl_is_rettype_inferred(li)) { + (void)jl_type_infer(&li, world, 0); + } } return 1; } @@ -1941,7 +1956,7 @@ static void show_call(jl_value_t *F, jl_value_t **args, uint32_t nargs) static jl_value_t *verify_type(jl_value_t *v) { - assert(jl_typeof(jl_typeof(v))); + assert(v && jl_typeof(v) && jl_typeof(jl_typeof(v)) == (jl_value_t*)jl_datatype_type); return v; } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index cc2f541bb8095..cd730e8d4c031 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -21,6 +21,14 @@ #include #endif #endif +// for outputting assembly +#include +#include +#include +#include +#include +#include +#include #include #include @@ -59,33 +67,18 @@ namespace llvm { #include #include #include -#include "codegen_shared.h" using namespace llvm; #include "julia.h" #include "julia_internal.h" +#include "codegen_shared.h" #include "jitlayers.h" #include "julia_assert.h" RTDyldMemoryManager* createRTDyldMemoryManager(void); -static IntegerType *T_uint32; -static IntegerType *T_uint64; -static IntegerType *T_size; -static Type *T_psize; -static Type *T_pjlvalue; -void jl_init_jit(Type *T_pjlvalue_) -{ - T_uint32 = Type::getInt32Ty(jl_LLVMContext); - T_uint64 = Type::getInt64Ty(jl_LLVMContext); - if (sizeof(size_t) == 8) - T_size = T_uint64; - else - T_size = T_uint32; - T_psize = PointerType::get(T_size, 0); - T_pjlvalue = T_pjlvalue_; -} +void jl_init_jit(void) { } // Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). @@ -269,6 +262,490 @@ void jl_add_optimization_passes(LLVMPassManagerRef PM, int opt_level) { addOptimizationPasses(unwrap(PM), opt_level); } +/// addPassesToX helper drives creation and initialization of TargetPassConfig. +#if JL_LLVM_VERSION >= 50000 +static MCContext * +addPassesToGenerateCode(LLVMTargetMachine *TM, PassManagerBase &PM) { + TargetPassConfig *PassConfig = TM->createPassConfig(PM); +#if JL_LLVM_VERSION < 60000 + PassConfig->setStartStopPasses(nullptr, nullptr, nullptr, nullptr); +#endif + PassConfig->setDisableVerify(false); + PM.add(PassConfig); + MachineModuleInfo *MMI = new MachineModuleInfo(TM); + PM.add(MMI); + if (PassConfig->addISelPasses()) + return NULL; + PassConfig->addMachinePasses(); + PassConfig->setInitialized(); + return &MMI->getContext(); +} +#else +static MCContext * +addPassesToGenerateCode(LLVMTargetMachine *TM, PassManagerBase &PM) { + // When in emulated TLS mode, add the LowerEmuTLS pass. + if (TM->Options.EmulatedTLS) + PM.add(createLowerEmuTLSPass(TM)); + + PM.add(createPreISelIntrinsicLoweringPass()); + + // Add internal analysis passes from the target machine. + PM.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); + + // Targets may override createPassConfig to provide a target-specific + // subclass. + TargetPassConfig *PassConfig = TM->createPassConfig(PM); + PassConfig->setStartStopPasses(nullptr, nullptr, nullptr); + + // Set PassConfig options provided by TargetMachine. + PassConfig->setDisableVerify(false); + + PM.add(PassConfig); + + PassConfig->addIRPasses(); + + PassConfig->addCodeGenPrepare(); + + PassConfig->addPassesToHandleExceptions(); + + PassConfig->addISelPrepare(); + + MachineModuleInfo &MMI = TM->addMachineModuleInfo(PM); + TM->addMachineFunctionAnalysis(PM, nullptr); + + // Disable FastISel with -O0 + TM->setO0WantsFastISel(false); + + if (PassConfig->addInstSelector()) + return nullptr; + + PassConfig->addMachinePasses(); + + PassConfig->setInitialized(); + + return &MMI.getContext(); +} +#endif + +// Snooping on which functions are being compiled, and how long it takes +JL_STREAM *dump_compiles_stream = NULL; +extern "C" JL_DLLEXPORT +void jl_dump_compiles(void *s) +{ + dump_compiles_stream = (JL_STREAM*)s; +} + +static void jl_add_to_ee(); +static uint64_t getAddressForFunction(StringRef fname); +extern "C" tracer_cb jl_linfo_tracer; + +void jl_jit_globals(std::map &globals) +{ + for (auto &global : globals) { + Constant *P = literal_static_pointer_val(global.first, global.second->getValueType()); + global.second->setInitializer(P); + global.second->setConstant(true); + global.second->setLinkage(GlobalValue::PrivateLinkage); + global.second->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + } +} + + +// this generates llvm code for the lambda info +// and adds the result to the jitlayers +// (and the shadow module), +// and generates code for it +static jl_generic_fptr_t _jl_compile_linfo( + jl_method_instance_t *li, + jl_code_info_t *src, + size_t world, + std::vector &triggered_linfos) +{ + // caller must hold codegen_lock + // and have disabled finalizers + JL_TIMING(CODEGEN); + uint64_t start_time = 0; + if (dump_compiles_stream != NULL) + start_time = jl_hrtime(); + + assert(jl_is_method_instance(li)); + assert(li->min_world <= world && (li->max_world >= world || li->max_world == 0) && + "invalid world for method-instance"); + assert(src && jl_is_code_info(src)); + + jl_generic_fptr_t fptr = {}; + // emit the code in LLVM IR form + jl_codegen_params_t params; + params.cache = true; + params.world = world; + std::map emitted; + jl_compile_result_t result = jl_compile_linfo1(li, src, params); + if (std::get<0>(result)) + emitted[li] = std::move(result); + jl_compile_workqueue(emitted, params); + + jl_jit_globals(params.globals); + for (auto &def : emitted) { + // Add the results to the execution engine now + jl_finalize_module(std::move(std::get<0>(def.second))); + } + jl_add_to_ee(); + for (auto &def : emitted) { + jl_method_instance_t *this_li = def.first; + jl_llvm_functions_t decls = std::get<1>(def.second); + jl_value_t *rettype = std::get<2>(def.second); + uint8_t api = std::get<3>(def.second); + jl_fptr_t addr = (jl_fptr_t)getAddressForFunction(decls.functionObject); + if (this_li->jlcall_api == JL_API_NOT_SET) { + this_li->fptr = addr; + this_li->jlcall_api = api; + } + if (!decls.specFunctionObject.empty() && jl_egal(rettype, this_li->rettype)) + this_li->fptr_specsig = (jl_fptr_t)getAddressForFunction(decls.specFunctionObject); + if (this_li->compile_traced) + triggered_linfos.push_back(this_li); + if (this_li == li) { + fptr.jlcall_api = api; + fptr.fptr = addr; + } + } + + uint64_t end_time = 0; + if (dump_compiles_stream != NULL) + end_time = jl_hrtime(); + + // If logging of the compilation stream is enabled, + // then dump the method-instance specialization type to the stream + if (dump_compiles_stream != NULL && jl_is_method(li->def.method)) { + jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", end_time - start_time); + jl_static_show(dump_compiles_stream, li->specTypes); + jl_printf(dump_compiles_stream, "\"\n"); + } + return fptr; +} + +void jl_strip_llvm_debug(Module *m); + +// get the address of a C-callable entry point for a function +extern "C" JL_DLLEXPORT +void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) +{ + JL_GC_PUSH1(&argt); + JL_LOCK(&codegen_lock); + jl_codegen_params_t params; + Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt, params); + jl_jit_globals(params.globals); + assert(params.workqueue.empty()); + jl_add_to_ee(); + JL_GC_POP(); + void *ptr = (void*)getAddressForFunction(llvmf->getName()); + JL_UNLOCK(&codegen_lock); + return ptr; +} + + +// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name +extern "C" JL_DLLEXPORT +void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) +{ + assert(jl_is_tuple_type(argt)); + JL_LOCK(&codegen_lock); + jl_codegen_params_t params; + Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt, params); + jl_jit_globals(params.globals); + assert(params.workqueue.empty()); + // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) + jl_add_to_ee(); + uint64_t Addr = getAddressForFunction(llvmf->getName()); + + if (imaging_mode) + llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); + + // make the alias to the shadow_module + GlobalAlias *GA = + GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); + + // make sure the alias name is valid for the current session + jl_ExecutionEngine->addGlobalMapping(GA, (void*)(uintptr_t)Addr); + JL_UNLOCK(&codegen_lock); +} + + +static jl_generic_fptr_t _jl_generate_fptr(jl_method_instance_t *li, jl_code_info_t *src, size_t world, bool force) +{ + jl_generic_fptr_t fptr = {}; + if (src && jl_is_code_info(src)) { + std::vector triggered_linfos; + jl_ptls_t ptls = jl_get_ptls_states(); + jl_gc_enable_finalizers(ptls, 0); // prevent any sort of unexpected recursion + // now we need to check again if it was compiled by recursion + fptr.jlcall_api = li->jlcall_api; + fptr.fptr = li->fptr; + if (force || !fptr.fptr || fptr.jlcall_api == JL_API_NOT_SET) { + fptr = _jl_compile_linfo(li, src, world, triggered_linfos); + } + jl_gc_enable_finalizers(ptls, 1); + for (jl_method_instance_t *linfo : triggered_linfos) // TODO: would be better to have released the locks first + if (jl_linfo_tracer) + jl_call_tracer(jl_linfo_tracer, (jl_value_t*)linfo); + } + return fptr; +} + +// this compiles li and emits fptr +extern "C" +jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t **pli, size_t world) +{ + jl_method_instance_t *li = *pli; + jl_generic_fptr_t fptr; + fptr.fptr = li->fptr; + fptr.jlcall_api = li->jlcall_api; + if (fptr.fptr && fptr.jlcall_api) { + return fptr; + } + JL_LOCK(&codegen_lock); + fptr.fptr = li->fptr; + fptr.jlcall_api = li->jlcall_api; + if (fptr.fptr && fptr.jlcall_api) { + JL_UNLOCK(&codegen_lock); + return fptr; + } + // if not already present, see if we have source code + // get a CodeInfo object to compile + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + if (!jl_is_method(li->def.method)) { + src = (jl_code_info_t*)li->inferred; + } + else { + // If the caller didn't provide the source, + // see if it is inferred + // or try to infer it for ourself + src = (jl_code_info_t*)li->inferred; + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(li->def.method, (jl_array_t*)src); + if (!src || !jl_is_code_info(src)) { + src = jl_type_infer(pli, world, 0); + li = *pli; + } + fptr.fptr = li->fptr; + fptr.jlcall_api = li->jlcall_api; + if (fptr.fptr && fptr.jlcall_api) { + JL_UNLOCK(&codegen_lock); // Might GC + JL_GC_POP(); + return fptr; + } + } + fptr = _jl_generate_fptr(li, src, world, false); + JL_UNLOCK(&codegen_lock); // Might GC + JL_GC_POP(); + return fptr; +} + +// this compiles li and emits fptr +extern "C" +jl_generic_fptr_t jl_generate_fptr_for_unspecialized(jl_method_instance_t *unspec) +{ + assert(jl_is_method(unspec->def.method)); + jl_generic_fptr_t fptr; + fptr.fptr = unspec->fptr; + fptr.jlcall_api = unspec->jlcall_api; + if (fptr.fptr && fptr.jlcall_api) { + return fptr; + } + JL_LOCK(&codegen_lock); + fptr.fptr = unspec->fptr; + fptr.jlcall_api = unspec->jlcall_api; + if (fptr.fptr && fptr.jlcall_api) { + JL_UNLOCK(&codegen_lock); // Might GC + return fptr; + } + jl_code_info_t *src = (jl_code_info_t*)unspec->def.method->source; + JL_GC_PUSH1(&src); + if (src == NULL) { + // TODO: this is wrong + assert(unspec->def.method->generator); + // TODO: jl_code_for_staged can throw + src = jl_code_for_staged(unspec); + } + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(unspec->def.method, (jl_array_t*)src); + fptr.fptr = unspec->fptr; + fptr.jlcall_api = unspec->jlcall_api; + if (!fptr.fptr || fptr.jlcall_api == JL_API_NOT_SET) { + fptr = _jl_generate_fptr(unspec, src, unspec->min_world, false); + } + JL_UNLOCK(&codegen_lock); // Might GC + JL_GC_POP(); + return fptr; +} + +// --- native code info, and dump function to IR and ASM --- +// Get pointer to llvm::Function instance, compiling if necessary +// for use in reflection from Julia. +// this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: +// misuse will leak memory or cause read-after-free +extern "C" JL_DLLEXPORT +void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params) +{ + if (jl_is_method(linfo->def.method) && linfo->def.method->source == NULL && + linfo->def.method->generator == NULL) { + // not a generic function + return NULL; + } + + // get the source code for this function + jl_code_info_t *src = (jl_code_info_t*)linfo->inferred; + JL_GC_PUSH1(&src); + if (!src || (jl_value_t*)src == jl_nothing) { + src = jl_type_infer(&linfo, world, 0); + if (!src && jl_is_method(linfo->def.method)) + src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source; + } + if ((jl_value_t*)src == jl_nothing) + src = NULL; + if (src && !jl_is_code_info(src) && jl_is_method(linfo->def.method)) + src = jl_uncompress_ast(linfo->def.method, (jl_array_t*)src); + + // emit this function into a new module + if (src && jl_is_code_info(src)) { + jl_codegen_params_t output; + output.world = world; + output.params = ¶ms; + std::unique_ptr m; + jl_llvm_functions_t decls; + jl_value_t *rettype; + uint8_t api; + JL_LOCK(&codegen_lock); + std::tie(m, decls, rettype, api) = jl_compile_linfo1(linfo, src, output); + + Function *F = NULL; + if (m) { + // if compilation succeeded, prepare to return the result + const std::string *fname; + if (!getwrapper && !decls.specFunctionObject.empty()) + fname = &decls.specFunctionObject; + else + fname = &decls.functionObject; + if (optimize) + jl_globalPM->run(*m.get()); + F = cast(m->getNamedValue(*fname)); + m.release(); // the return object `llvmf` will be the owning pointer + } + JL_GC_POP(); + JL_UNLOCK(&codegen_lock); // Might GC + if (F) + return F; + } + + const char *mname = name_from_method_instance(linfo); + jl_errorf("unable to compile source for function %s", mname); +} + + + +// get a native disassembly for linfo +extern "C" JL_DLLEXPORT +jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world, + int raw_mc, char getwrapper, const char* asm_variant) +{ + // printing via disassembly + jl_generic_fptr_t fptr = {}; + fptr = jl_generate_fptr(&linfo, world); + if (fptr.jlcall_api == JL_API_CONST) { + // normally we don't generate native code for these functions, so need an exception here + // and an extra cache to remember the result + JL_LOCK(&codegen_lock); + static DenseMap cache; + auto &gen = cache[linfo]; + if (gen == NULL) { + jl_code_info_t *src = jl_type_infer(&linfo, world, 0); + JL_GC_PUSH1(&src); + if (jl_is_method(linfo->def.method)) { + if (!src) { + // TODO: jl_code_for_staged can throw + src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source; + } + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(linfo->def.method, (jl_array_t*)src); + } + fptr.fptr = linfo->fptr; + fptr.jlcall_api = linfo->jlcall_api; + if (fptr.jlcall_api == JL_API_CONST || fptr.jlcall_api == JL_API_NOT_SET) { + fptr = _jl_generate_fptr(linfo, src, world, true); + } + if (fptr.jlcall_api != JL_API_CONST && fptr.jlcall_api != JL_API_NOT_SET) { + gen = fptr.fptr; + } + JL_GC_POP(); + } + JL_UNLOCK(&codegen_lock); + } + + jl_fptr_t specfptr = linfo->fptr_specsig; + if (!getwrapper && specfptr) { + fptr.fptr = specfptr; + fptr.jlcall_api = JL_API_GENERIC; + } + + if (fptr.jlcall_api != JL_API_NOT_SET && fptr.jlcall_api != JL_API_CONST && fptr.fptr) + return jl_dump_fptr_asm((uint64_t)(uintptr_t)fptr.fptr, raw_mc, asm_variant); + + // precise printing via IR assembler + SmallVector ObjBufferSV; + { // scope block + llvm::raw_svector_ostream asmfile(ObjBufferSV); + Function *f = (Function*)jl_get_llvmf_defn(linfo, world, getwrapper, true, jl_default_cgparams); + assert(!f->isDeclaration()); + std::unique_ptr m(f->getParent()); + for (auto &f2 : m->functions()) { + if (f != &f2 && !f->isDeclaration()) + f2.deleteBody(); + } + jl_strip_llvm_debug(m.get()); + legacy::PassManager PM; + LLVMTargetMachine *TM = static_cast(jl_TargetMachine); + MCContext *Context = addPassesToGenerateCode(TM, PM); + if (Context) { +#if JL_LLVM_VERSION >= 60000 + const MCSubtargetInfo &STI = *TM->getMCSubtargetInfo(); +#endif + const MCAsmInfo &MAI = *TM->getMCAsmInfo(); + const MCRegisterInfo &MRI = *TM->getMCRegisterInfo(); + const MCInstrInfo &MII = *TM->getMCInstrInfo(); + unsigned OutputAsmDialect = MAI.getAssemblerDialect(); + if (!strcmp(asm_variant, "att")) + OutputAsmDialect = 0; + if (!strcmp(asm_variant, "intel")) + OutputAsmDialect = 1; + MCInstPrinter *InstPrinter = TM->getTarget().createMCInstPrinter( + TM->getTargetTriple(), OutputAsmDialect, MAI, MII, MRI); + MCAsmBackend *MAB = TM->getTarget().createMCAsmBackend( +#if JL_LLVM_VERSION >= 60000 + STI, MRI, TM->Options.MCOptions +#elif JL_LLVM_VERSION >= 40000 + MRI, TM->getTargetTriple().str(), TM->getTargetCPU(), TM->Options.MCOptions +#else + MRI, TM->getTargetTriple().str(), TM->getTargetCPU() +#endif + ); + auto FOut = llvm::make_unique(asmfile); + MCCodeEmitter *MCE = nullptr; + std::unique_ptr S(TM->getTarget().createAsmStreamer( + *Context, std::move(FOut), true, + true, InstPrinter, MCE, MAB, false)); + AsmPrinter *Printer = + TM->getTarget().createAsmPrinter(*TM, std::move(S)); + if (Printer) { + PM.add(Printer); + PM.run(*m); + } + } + } + return jl_pchar_to_string(ObjBufferSV.data(), ObjBufferSV.size()); +} + // ------------------------ TEMPORARILY COPIED FROM LLVM ----------------- // This must be kept in sync with gdb/gdb/jit.h . extern "C" { @@ -390,15 +867,14 @@ void JuliaOJIT::DebugObjectRegistrar::registerObject(RTDyldObjHandleT H, const O NotifyGDB(SavedObject); } + object::ObjectFile *SavedBinary = SavedObject.getBinary(); SavedObjects.push_back(std::move(SavedObject)); - ORCNotifyObjectEmitted(JuliaListener.get(), *Object, - *SavedObjects.back().getBinary(), - *LO, JIT.MemMgr.get()); + ORCNotifyObjectEmitted(JuliaListener.get(), *Object, *SavedBinary, *LO, JIT.MemMgr.get()); // record all of the exported symbols defined in this object // in the primary hash table for the enclosing JIT - for (auto &Symbol : Object->symbols()) { + for (auto &Symbol : SavedBinary->symbols()) { auto Flags = Symbol.getFlags(); if (Flags & object::BasicSymbolRef::SF_Undefined) continue; @@ -413,10 +889,11 @@ void JuliaOJIT::DebugObjectRegistrar::registerObject(RTDyldObjHandleT H, const O // as an alternative, we could store the JITSymbol instead // (which would present a lazy-initializer functor interface instead) #if JL_LLVM_VERSION >= 50000 - JIT.LocalSymbolTable[Name] = (void*)(uintptr_t)cantFail(Sym.getAddress()); + void *addr = (void*)(uintptr_t)cantFail(Sym.getAddress()); #else - JIT.LocalSymbolTable[Name] = (void*)(uintptr_t)Sym.getAddress(); + void *addr = (void*)(uintptr_t)Sym.getAddress(); #endif + JIT.LocalSymbolTable[Name] = addr; } } @@ -538,6 +1015,12 @@ void *JuliaOJIT::getPointerToGlobalIfAvailable(const GlobalValue *GV) void JuliaOJIT::addModule(std::unique_ptr M) { + std::vector NewExports; + for (auto &F : M->functions()) { + if (!F.isDeclaration() && F.getLinkage() == GlobalValue::ExternalLinkage) { + NewExports.push_back(strdup(F.getName().str().c_str())); + } + } #ifndef JL_NDEBUG // validate the relocations for M for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { @@ -605,6 +1088,11 @@ void JuliaOJIT::addModule(std::unique_ptr M) #else CompileLayer.emitAndFinalize(modset); #endif + // record a stable name for this fptr address + for (auto Name : NewExports) { + void *addr = LocalSymbolTable[getMangledName(Name)]; + ReverseLocalSymbolTable[addr] = Name; + } } void JuliaOJIT::removeModule(ModuleHandleT H) @@ -624,8 +1112,11 @@ JL_JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbols Addr = getPointerToGlobalIfAvailable(Name); } // Step 2: Search all previously emitted symbols - if (Addr == nullptr) - Addr = LocalSymbolTable[Name]; + if (Addr == nullptr) { + auto it = LocalSymbolTable.find(Name); + if (it != LocalSymbolTable.end()) + Addr = it->second; + } return JL_JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } @@ -654,11 +1145,30 @@ uint64_t JuliaOJIT::getFunctionAddress(const std::string &Name) #endif } -Function *JuliaOJIT::FindFunctionNamed(const std::string &Name) +StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_method_instance_t *li) { - return shadow_output->getFunction(Name); + auto &fname = ReverseLocalSymbolTable[(void*)(uintptr_t)Addr]; + if (fname.empty()) { + std::stringstream stream_fname; + if (li->fptr_specsig) { + if (Addr == (uint64_t)(uintptr_t)li->fptr_specsig) + stream_fname << "jlsys_"; // the specsig implementation + else + stream_fname << "jlsysw_"; // it's a specsig wrapper + } + else { + stream_fname << "jsys1_"; // it's a jlcall without a specsig + } + const char* unadorned_name = jl_symbol_name(li->def.method->name); + stream_fname << unadorned_name << "_" << globalUnique++; + std::string string_fname = stream_fname.str(); + fname = strdup(string_fname.c_str()); + LocalSymbolTable[getMangledName(string_fname)] = (void*)(uintptr_t)Addr; + } + return fname; } + void JuliaOJIT::RegisterJITEventListener(JITEventListener *L) { // TODO @@ -674,7 +1184,7 @@ const Triple& JuliaOJIT::getTargetTriple() const return TM.getTargetTriple(); } -std::string JuliaOJIT::getMangledName(const std::string &Name) +std::string JuliaOJIT::getMangledName(StringRef Name) { SmallString<128> FullName; Mangler::getNameWithPrefix(FullName, Name, DL); @@ -688,53 +1198,17 @@ std::string JuliaOJIT::getMangledName(const GlobalValue *GV) JuliaOJIT *jl_ExecutionEngine; -// MSVC's link.exe requires each function declaration to have a Comdat section -// So rather than litter the code with conditionals, -// all global values that get emitted call this function -// and it decides whether the definition needs a Comdat section and adds the appropriate declaration -// TODO: consider moving this into jl_add_to_shadow or jl_dump_shadow? the JIT doesn't care, so most calls are now no-ops -template // for GlobalObject's -static T *addComdat(T *G) -{ -#if defined(_OS_WINDOWS_) - if (imaging_mode && !G->isDeclaration()) { - // Add comdat information to make MSVC link.exe happy - // it's valid to emit this for ld.exe too, - // but makes it very slow to link for no benefit - if (G->getParent() == shadow_output) { -#if defined(_COMPILER_MICROSOFT_) - Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); - // ELF only supports Comdat::Any - jl_Comdat->setSelectionKind(Comdat::NoDuplicates); - G->setComdat(jl_Comdat); -#endif -#if defined(_CPU_X86_64_) - // Add unwind exception personalities to functions to handle async exceptions - assert(!juliapersonality_func || juliapersonality_func->getParent() == shadow_output); - if (Function *F = dyn_cast(G)) - F->setPersonalityFn(juliapersonality_func); -#endif - } - // add __declspec(dllexport) to everything marked for export - if (G->getLinkage() == GlobalValue::ExternalLinkage) - G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); - else - G->setDLLStorageClass(GlobalValue::DefaultStorageClass); - } -#endif - return G; -} - // destructively move the contents of src into dest // this assumes that the targets of the two modules are the same // including the DataLayout and ModuleFlags (for example) // and that there is no module-level assembly -static void jl_merge_module(Module *dest, std::unique_ptr src) +// Comdat is also removed, since the JIT doesn't need it +void jl_merge_module(Module *dest, std::unique_ptr src) { assert(dest != src.get()); for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) { GlobalVariable *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + GlobalVariable *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; // Replace a declaration with the definition: if (dG) { @@ -744,6 +1218,7 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) continue; } else { + assert(dG->isDeclaration() || dG->getInitializer() == sG->getInitializer()); dG->replaceAllUsesWith(sG); dG->eraseFromParent(); } @@ -751,13 +1226,13 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) // Reparent the global variable: sG->removeFromParent(); dest->getGlobalList().push_back(sG); - // Comdat is owned by the Module, recreate it in the new parent: - addComdat(sG); + // Comdat is owned by the Module + sG->setComdat(nullptr); } for (Module::iterator I = src->begin(), E = src->end(); I != E;) { Function *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + Function *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; // Replace a declaration with the definition: if (dG) { @@ -767,6 +1242,7 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) continue; } else { + assert(dG->isDeclaration()); dG->replaceAllUsesWith(sG); dG->eraseFromParent(); } @@ -774,13 +1250,13 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) // Reparent the global variable: sG->removeFromParent(); dest->getFunctionList().push_back(sG); - // Comdat is owned by the Module, recreate it in the new parent: - addComdat(sG); + // Comdat is owned by the Module + sG->setComdat(nullptr); } for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) { GlobalAlias *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + GlobalAlias *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; if (dG) { if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two @@ -808,114 +1284,45 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) } } -// to finalize a function, look up its name in the `module_for_fname` map of -// unfinalized functions and merge it, plus any other modules it depends upon, -// into `collector` then add `collector` to the execution engine -static StringMap module_for_fname; -static void jl_merge_recursive(Module *m, Module *collector); +// this is a unique_ptr, but we don't want +// C++ to attempt to run out finalizer on exit +// since it also owned by jl_LLVMContext +static Module *ready_to_emit; -static void jl_add_to_ee(std::unique_ptr m) +static void jl_add_to_ee() { + JL_TIMING(LLVM_EMIT); + std::unique_ptr m(ready_to_emit); + ready_to_emit = NULL; + if (m) { #if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) - // Add special values used by debuginfo to build the UnwindData table registration for Win64 - ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); + // Add special values used by debuginfo to build the UnwindData table registration for Win64 + Type *T_uint32 = Type::getInt32Ty(m->getContext()); + ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); #endif - assert(jl_ExecutionEngine); - jl_ExecutionEngine->addModule(std::move(m)); -} - -void jl_finalize_function(StringRef F) -{ - std::unique_ptr m(module_for_fname.lookup(F)); - if (m) { - jl_merge_recursive(m.get(), m.get()); - jl_add_to_ee(std::move(m)); - } -} - -static void jl_finalize_function(const std::string &F, Module *collector) -{ - std::unique_ptr m(module_for_fname.lookup(F)); - if (m) { - jl_merge_recursive(m.get(), collector); - jl_merge_module(collector, std::move(m)); - } -} - -static void jl_merge_recursive(Module *m, Module *collector) -{ - // probably not many unresolved declarations, but be sure to iterate over their Names, - // since the declarations may get destroyed by the jl_merge_module call. - // this is also why we copy the Name string, rather than save a StringRef - SmallVector to_finalize; - for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { - Function *F = &*I; - if (!F->isDeclaration()) { - module_for_fname.erase(F->getName()); - } - else if (!isIntrinsicFunction(F)) { - to_finalize.push_back(F->getName().str()); - } - } - - for (const auto F : to_finalize) { - jl_finalize_function(F, collector); + assert(jl_ExecutionEngine); + jl_ExecutionEngine->addModule(std::move(m)); } } -// see if any of the functions needed by F are still WIP -static StringSet<> incomplete_fname; -static bool can_finalize_function(StringRef F, SmallSet &known) +// this passes ownership of a module to the JIT after code emission is complete +void jl_finalize_module(std::unique_ptr m) { - if (incomplete_fname.find(F) != incomplete_fname.end()) - return false; - Module *M = module_for_fname.lookup(F); - if (M && known.insert(M).second) { - for (Module::iterator I = M->begin(), E = M->end(); I != E; ++I) { - Function *F = &*I; - if (F->isDeclaration() && !isIntrinsicFunction(F)) { - if (!can_finalize_function(F->getName(), known)) - return false; - } - } - } - return true; -} -bool jl_can_finalize_function(StringRef F) -{ - SmallSet known; - return can_finalize_function(F, known); -} - -// let the JIT know this function is a WIP -void jl_init_function(Function *F) -{ - incomplete_fname.insert(F->getName()); + if (ready_to_emit) + jl_merge_module(ready_to_emit, std::move(m)); + else + ready_to_emit = m.release(); } -// this takes ownership of a module after code emission is complete -// and will add it to the execution engine when required (by jl_finalize_function) -void jl_finalize_module(Module *m, bool shadow) +static uint64_t getAddressForFunction(StringRef fname) { - // record the function names that are part of this Module - // so it can be added to the JIT when needed - for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { - Function *F = &*I; - if (!F->isDeclaration()) { - bool known = incomplete_fname.erase(F->getName()); - (void)known; // TODO: assert(known); // llvmcall gets this wrong - module_for_fname[F->getName()] = m; - } - } - // in the newer JITs, the shadow module is separate from the execution module - if (shadow) - jl_add_to_shadow(m); + return jl_ExecutionEngine->getFunctionAddress(fname); } // helper function for adding a DLLImport (dlsym) address to the execution engine @@ -935,277 +1342,9 @@ void add_named_global(GlobalObject *gv, void *addr, bool dllimport) jl_ExecutionEngine->addGlobalMapping(gv, addr); } -static std::vector jl_sysimg_gvars; -static std::vector jl_sysimg_fvars; -static std::map jl_value_to_llvm; - -// global variables to pointers are pretty common, -// so this method is available as a convenience for emitting them. -// for other types, the formula for implementation is straightforward: -// (see stringConstPtr, for an alternative example to the code below) -// -// if in imaging_mode, emit a GlobalVariable with the same name and an initializer to the shadow_module -// making it valid for emission and reloading in the sysimage -// -// then add a global mapping to the current value (usually from calloc'd space) -// to the execution engine to make it valid for the current session (with the current value) -void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit) -{ - PointerType *T = cast(gv->getType()->getElementType()); // pointer is the only supported type here - - GlobalVariable *shadowvar = NULL; - if (imaging_mode) - shadowvar = global_proto(gv, shadow_output); - - if (shadowvar) { - shadowvar->setInitializer(ConstantPointerNull::get(T)); - shadowvar->setLinkage(GlobalVariable::InternalLinkage); - addComdat(shadowvar); - if (imaging_mode && gvarinit) { - // make the pointer valid for future sessions - jl_sysimg_gvars.push_back(shadowvar); - jl_value_llvm gv_struct; - gv_struct.gv = global_proto(gv); - gv_struct.index = jl_sysimg_gvars.size(); - jl_value_to_llvm[gvarinit] = gv_struct; - } - } - - // make the pointer valid for this session - void *slot = calloc(1, sizeof(void*)); - jl_ExecutionEngine->addGlobalMapping(gv, slot); - return slot; -} - -void* jl_get_globalvar(GlobalVariable *gv) -{ - void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(gv); - assert(p); - return p; -} - -// clones the contents of the module `m` to the shadow_output collector -void jl_add_to_shadow(Module *m) -{ -#ifndef KEEP_BODIES - if (!imaging_mode && !jl_options.outputjitbc) - return; -#endif - ValueToValueMapTy VMap; - std::unique_ptr clone(CloneModule(m, VMap)); - for (Module::iterator I = clone->begin(), E = clone->end(); I != E; ++I) { - Function *F = &*I; - if (!F->isDeclaration()) { - F->setLinkage(Function::InternalLinkage); - addComdat(F); - } - } - jl_merge_module(shadow_output, std::move(clone)); -} - -static void emit_offset_table(Module *mod, const std::vector &vars, StringRef name) -{ - // Emit a global variable with all the variable addresses. - // The cloning pass will convert them into offsets. - assert(!vars.empty()); - size_t nvars = vars.size(); - std::vector addrs(nvars); - for (size_t i = 0; i < nvars; i++) - addrs[i] = ConstantExpr::getBitCast(vars[i], T_psize); - ArrayType *vars_type = ArrayType::get(T_psize, nvars); - new GlobalVariable(*mod, vars_type, true, - GlobalVariable::ExternalLinkage, - ConstantArray::get(vars_type, addrs), - name); -} - - -static void jl_gen_llvm_globaldata(Module *mod, const char *sysimg_data, size_t sysimg_len) -{ - emit_offset_table(mod, jl_sysimg_gvars, "jl_sysimg_gvars"); - emit_offset_table(mod, jl_sysimg_fvars, "jl_sysimg_fvars"); - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, globalUnique+1), - "jl_globalUnique")); - - // reflect the address of the jl_RTLD_DEFAULT_handle variable - // back to the caller, so that we can check for consistency issues - GlobalValue *jlRTLD_DEFAULT_var = mod->getNamedValue("jl_RTLD_DEFAULT_handle"); - addComdat(new GlobalVariable(*mod, - jlRTLD_DEFAULT_var->getType(), - true, - GlobalVariable::ExternalLinkage, - jlRTLD_DEFAULT_var, - "jl_RTLD_DEFAULT_handle_pointer")); - - if (sysimg_data) { - Constant *data = ConstantDataArray::get(jl_LLVMContext, - ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); - addComdat(new GlobalVariable(*mod, data->getType(), false, - GlobalVariable::ExternalLinkage, - data, "jl_system_image_data"))->setAlignment(64); - Constant *len = ConstantInt::get(T_size, sysimg_len); - addComdat(new GlobalVariable(*mod, len->getType(), true, - GlobalVariable::ExternalLinkage, - len, "jl_system_image_size")); - } -} - -// takes the running content that has collected in the shadow module and dump it to disk -// this builds the object file portion of the sysimage files for fast startup -extern "C" -void jl_dump_native(const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len) -{ - JL_TIMING(NATIVE_DUMP); - // We don't want to use MCJIT's target machine because - // it uses the large code model and we may potentially - // want less optimizations there. - Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); - // make sure to emit the native object format, even if FORCE_ELF was set in codegen -#if defined(_OS_WINDOWS_) - TheTriple.setObjectFormat(Triple::COFF); -#elif defined(_OS_DARWIN_) - TheTriple.setObjectFormat(Triple::MachO); - TheTriple.setOS(llvm::Triple::MacOSX); -#endif - std::unique_ptr - TM(jl_TargetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - jl_TargetMachine->getTargetCPU(), - jl_TargetMachine->getTargetFeatureString(), - jl_TargetMachine->Options, -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - Reloc::PIC_, -#else - Optional(), -#endif - // Use small model so that we can use signed 32bits offset in the function and GV tables - CodeModel::Small, - CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? - )); - - legacy::PassManager PM; - addTargetPasses(&PM, TM.get()); - - // set up optimization passes - std::unique_ptr unopt_bc_OS; - std::unique_ptr bc_OS; - std::unique_ptr obj_OS; - - if (unopt_bc_fname) { - // call output handler directly to avoid special case handling of `-` filename - int FD; - std::error_code EC = sys::fs::openFileForWrite(unopt_bc_fname, FD, sys::fs::F_None); - unopt_bc_OS.reset(new raw_fd_ostream(FD, true)); - std::string err; - if (EC) - err = "ERROR: failed to open --output-unopt-bc file '" + std::string(unopt_bc_fname) + "': " + EC.message(); - if (!err.empty()) - jl_safe_printf("%s\n", err.c_str()); - else { - PM.add(createBitcodeWriterPass(*unopt_bc_OS.get())); - } - } - - if (bc_fname || obj_fname) - addOptimizationPasses(&PM, jl_options.opt_level, true); - - if (bc_fname) { - // call output handler directly to avoid special case handling of `-` filename - int FD; - std::error_code EC = sys::fs::openFileForWrite(bc_fname, FD, sys::fs::F_None); - bc_OS.reset(new raw_fd_ostream(FD, true)); - std::string err; - if (EC) - err = "ERROR: failed to open --output-bc file '" + std::string(bc_fname) + "': " + EC.message(); - if (!err.empty()) - jl_safe_printf("%s\n", err.c_str()); - else { - PM.add(createBitcodeWriterPass(*bc_OS.get())); - } - } - - if (obj_fname) { - // call output handler directly to avoid special case handling of `-` filename - int FD; - std::error_code EC = sys::fs::openFileForWrite(obj_fname, FD, sys::fs::F_None); - obj_OS.reset(new raw_fd_ostream(FD, true)); - std::string err; - if (EC) - err = "ERROR: failed to open --output-o file '" + std::string(obj_fname) + "': " + EC.message(); - if (!err.empty()) - jl_safe_printf("%s\n", err.c_str()); - else { - if (TM->addPassesToEmitFile(PM, *obj_OS.get(), TargetMachine::CGFT_ObjectFile, false)) { - jl_safe_printf("ERROR: target does not support generation of object files\n"); - } - } - } - - // Reset the target triple to make sure it matches the new target machine - shadow_output->setTargetTriple(TM->getTargetTriple().str()); -#if JL_LLVM_VERSION >= 40000 - DataLayout DL = TM->createDataLayout(); - DL.reset(DL.getStringRepresentation() + "-ni:10:11:12"); - shadow_output->setDataLayout(DL); -#else - shadow_output->setDataLayout(TM->createDataLayout()); -#endif - - // add metadata information - if (imaging_mode) - jl_gen_llvm_globaldata(shadow_output, sysimg_data, sysimg_len); - - // do the actual work - PM.run(*shadow_output); - imaging_mode = false; -} - -extern "C" int32_t jl_assign_functionID(const char *fname) -{ - // give the function an index in the constant lookup table - assert(imaging_mode); - if (fname == NULL) - return 0; - jl_sysimg_fvars.push_back(shadow_output->getNamedValue(fname)); - return jl_sysimg_fvars.size(); -} - -extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) -{ - // map a jl_value_t memory location to a GlobalVariable - std::map::iterator it; - it = jl_value_to_llvm.find(p); - if (it == jl_value_to_llvm.end()) - return 0; - return it->second.index; -} - -GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M) -{ - // emit a GlobalVariable for a jl_value_t named "cname" - std::map::iterator it; - // first see if there already is a GlobalVariable for this address - it = jl_value_to_llvm.find(addr); - if (it != jl_value_to_llvm.end()) - return prepare_global_in(M, (llvm::GlobalVariable*)it->second.gv); - - std::stringstream gvname; - gvname << cname << globalUnique++; - // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(*M, T_pjlvalue, - false, GlobalVariable::ExternalLinkage, - NULL, gvname.str()); - *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; - return gv; -} // An LLVM module pass that just runs all julia passes in order. Useful for // debugging -extern "C" void jl_init_codegen(void); template class JuliaPipeline : public Pass { public: @@ -1217,7 +1356,7 @@ class JuliaPipeline : public Pass { void add(Pass *P) { TPM->schedulePass(P); } }; void preparePassManager(PMStack &Stack) override { - (void)jl_init_llvm(); + jl_init_llvm(); PMTopLevelManager *TPM = Stack.top()->getTopLevelManager(); TPMAdapter Adapter(TPM); addTargetPasses(&Adapter, jl_TargetMachine); @@ -1234,3 +1373,20 @@ template<> char JuliaPipeline<3>::ID = 0; static RegisterPass> X("juliaO0", "Runs the entire julia pipeline (at -O0)", false, false); static RegisterPass> Y("julia", "Runs the entire julia pipeline (at -O2)", false, false); static RegisterPass> Z("juliaO3", "Runs the entire julia pipeline (at -O3)", false, false); + +extern "C" JL_DLLEXPORT +uint64_t jl_get_llvm_fptr(void *function) +{ + Function *F = (Function*)function; + uint64_t addr = getAddressForFunction(F->getName()); + if (!addr) { +#if JL_LLVM_VERSION >= 50000 + if (auto exp_addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress()) { + addr = exp_addr.get(); + } +#else + addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress(); +#endif + } + return addr; +} diff --git a/src/jitlayers.h b/src/jitlayers.h index 2b9041dc26af5..49cba6746a01f 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -35,18 +35,55 @@ extern Function *juliapersonality_func; #endif -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing - void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM); void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump_native=false); -void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL); -void* jl_get_globalvar(GlobalVariable *gv); -GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M); -void jl_add_to_shadow(Module *m); -void jl_init_function(Function *f); -bool jl_can_finalize_function(StringRef F); -void jl_finalize_function(StringRef F); -void jl_finalize_module(Module *m, bool shadow); +void jl_finalize_module(std::unique_ptr m); +void jl_merge_module(Module *dest, std::unique_ptr src); + +typedef struct _jl_llvm_functions_t { + std::string functionObject; // jlcall llvm Function name + std::string specFunctionObject; // specialized llvm Function name +} jl_llvm_functions_t; + +struct jl_returninfo_t { + llvm::Function *decl; + enum CallingConv { + Boxed = 0, + Register, + SRet, + Union, + Ghosts + } cc; + size_t union_bytes; + size_t union_align; + size_t union_minalign; +}; + +typedef std::vector> jl_codegen_call_targets_t; +typedef std::tuple, jl_llvm_functions_t, jl_value_t*, uint8_t> jl_compile_result_t; + +typedef struct { + // outputs + jl_codegen_call_targets_t workqueue; + std::map globals; + // inputs + size_t world = 0; + const jl_cgparams_t *params = &jl_default_cgparams; + bool cache = false; +} jl_codegen_params_t; + +jl_compile_result_t jl_compile_linfo1( + jl_method_instance_t *li, + jl_code_info_t *src, + jl_codegen_params_t ¶ms); + +void jl_compile_workqueue( + std::map &emitted, + jl_codegen_params_t ¶ms); + +Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt, + jl_codegen_params_t ¶ms); + // Connect Modules via prototypes, each owned by module `M` static inline GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) @@ -84,7 +121,26 @@ static inline void add_named_global(GlobalObject *gv, T *addr, bool dllimport = add_named_global(gv, (void*)(uintptr_t)addr, dllimport); } -void jl_init_jit(Type *T_pjlvalue_); +static inline Constant *literal_static_pointer_val(const void *p, Type *T) +{ + // this function will emit a static pointer into the generated code + // the generated code will only be valid during the current session, + // and thus, this should typically be avoided in new API's +#if defined(_P64) + return ConstantExpr::getIntToPtr(ConstantInt::get(Type::getInt64Ty(T->getContext()), (uint64_t)p), T); +#else + return ConstantExpr::getIntToPtr(ConstantInt::get(Type::getInt32Ty(T->getContext()), (uint32_t)p), T); +#endif +} + +static const inline char *name_from_method_instance(jl_method_instance_t *li) +{ + return jl_is_method(li->def.method) ? jl_symbol_name(li->def.method->name) : "top-level scope"; +} + + +void jl_init_jit(void); + #if JL_LLVM_VERSION >= 40000 typedef JITSymbol JL_JITSymbol; // The type that is similar to SymbolInfo on LLVM 4.0 is actually @@ -152,12 +208,12 @@ class JuliaOJIT { JL_JITSymbol findUnmangledSymbol(const std::string Name); uint64_t getGlobalValueAddress(const std::string &Name); uint64_t getFunctionAddress(const std::string &Name); - Function *FindFunctionNamed(const std::string &Name); + StringRef getFunctionAtAddress(uint64_t Addr, jl_method_instance_t *li); void RegisterJITEventListener(JITEventListener *L); const DataLayout& getDataLayout() const; const Triple& getTargetTriple() const; private: - std::string getMangledName(const std::string &Name); + std::string getMangledName(StringRef Name); std::string getMangledName(const GlobalValue *GV); TargetMachine &TM; @@ -174,9 +230,9 @@ class JuliaOJIT { CompileLayerT CompileLayer; SymbolTableT GlobalSymbolTable; SymbolTableT LocalSymbolTable; + DenseMap ReverseLocalSymbolTable; }; extern JuliaOJIT *jl_ExecutionEngine; -JL_DLLEXPORT extern LLVMContext jl_LLVMContext; Pass *createLowerPTLSPass(bool imaging_mode); Pass *createCombineMulAddPass(); diff --git a/src/jloptions.c b/src/jloptions.c index fba335551fb56..db44700729916 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -63,7 +63,6 @@ jl_options_t jl_options = { 0, // quiet NULL, // bind-to NULL, // output-bc NULL, // output-unopt-bc - NULL, // output-jit-bc NULL, // output-o NULL, // output-ji 0, // incremental @@ -151,7 +150,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_code_coverage, opt_track_allocation, opt_check_bounds, - opt_output_jit_bc, opt_output_unopt_bc, opt_output_bc, opt_depwarn, @@ -204,7 +202,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "check-bounds", required_argument, 0, opt_check_bounds }, { "output-bc", required_argument, 0, opt_output_bc }, { "output-unopt-bc", required_argument, 0, opt_output_unopt_bc }, - { "output-jit-bc", required_argument, 0, opt_output_jit_bc }, { "output-o", required_argument, 0, opt_output_o }, { "output-ji", required_argument, 0, opt_output_ji }, { "output-incremental",required_argument, 0, opt_incremental }, @@ -502,9 +499,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_options.outputbc = optarg; if (!jl_options.image_file_specified) jl_options.image_file = NULL; break; - case opt_output_jit_bc: - jl_options.outputjitbc = optarg; - break; case opt_output_unopt_bc: jl_options.outputunoptbc = optarg; if (!jl_options.image_file_specified) jl_options.image_file = NULL; diff --git a/src/jltypes.c b/src/jltypes.c index 8d790c675b0e6..7bc2b86804245 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -129,7 +129,7 @@ jl_value_t *jl_memory_exception; jl_value_t *jl_readonlymemory_exception; union jl_typemap_t jl_cfunction_list; -jl_cgparams_t jl_default_cgparams = {1, 1, 1, 1, 0, NULL, NULL, NULL}; +jl_cgparams_t jl_default_cgparams = {1, 1, 1, 0, NULL, NULL, NULL}; // --- type properties and predicates --- @@ -2071,7 +2071,7 @@ void jl_init_types(void) jl_method_instance_type = jl_new_datatype(jl_symbol("MethodInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(16, + jl_perm_symsvec(14, "def", "specTypes", "rettype", @@ -2085,9 +2085,8 @@ void jl_init_types(void) "jlcall_api", "", "fptr", - "unspecialized_ducttape", - "", ""), - jl_svec(16, + "fptr_specsig"), + jl_svec(14, jl_new_struct(jl_uniontype_type, jl_method_type, jl_module_type), jl_any_type, jl_any_type, @@ -2101,8 +2100,7 @@ void jl_init_types(void) jl_uint8_type, jl_bool_type, jl_any_type, // void* - jl_any_type, // void* - jl_any_type, jl_any_type), // void*, void* + jl_any_type), // void* 0, 1, 4); // all kinds of types share a method table @@ -2185,8 +2183,6 @@ void jl_init_types(void) jl_svecset(jl_method_type->types, 10, jl_method_instance_type); jl_svecset(jl_method_instance_type->types, 12, jl_voidpointer_type); jl_svecset(jl_method_instance_type->types, 13, jl_voidpointer_type); - jl_svecset(jl_method_instance_type->types, 14, jl_voidpointer_type); - jl_svecset(jl_method_instance_type->types, 15, jl_voidpointer_type); jl_compute_field_offsets(jl_datatype_type); jl_compute_field_offsets(jl_typename_type); diff --git a/src/julia.h b/src/julia.h index 16c3ac43699f9..c51ef9bee8337 100644 --- a/src/julia.h +++ b/src/julia.h @@ -221,18 +221,13 @@ JL_EXTENSION typedef struct { union { jl_fptr_t fptr; jl_fptr_t fptr1; - // constant fptr2; + jl_value_t *constant_2; jl_fptr_sparam_t fptr3; jl_fptr_linfo_t fptr4; }; uint8_t jlcall_api; } jl_generic_fptr_t; -typedef struct _jl_llvm_functions_t { - const char *functionObject; // jlcall llvm Function name - const char *specFunctionObject; // specialized llvm Function name -} jl_llvm_functions_t; - // This type describes a single function body typedef struct _jl_code_info_t { jl_array_t *code; // Any array of statements @@ -307,13 +302,10 @@ typedef struct _jl_method_instance_t { size_t max_world; uint8_t inInference; // flags to tell if inference is running on this function uint8_t jlcall_api; - uint8_t compile_traced; // if set will notify callback if this linfo is compiled - jl_fptr_t fptr; // jlcall entry point with api specified by jlcall_api - jl_fptr_t unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred fptr may get stored here, jlcall_api = JL_API_GENERIC - - // names of declarations in the JIT, - // suitable for referencing in LLVM IR - jl_llvm_functions_t functionObjectsDecls; + uint8_t compile_traced; // if set, will notify callback if this linfo is compiled + // jlcall entry point with api specified by jlcall_api + jl_fptr_t fptr; + jl_fptr_t fptr_specsig; } jl_method_instance_t; // all values are callable as Functions @@ -1077,6 +1069,9 @@ STATIC_INLINE int jl_is_concrete_type(jl_value_t *v) JL_NOTSAFEPOINT return jl_is_datatype(v) && ((jl_datatype_t*)v)->isconcretetype; } +JL_DLLEXPORT int jl_is_cacheable_sig( + jl_tupletype_t *type, jl_tupletype_t *decl, jl_method_t *definition); + // type constructors JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *inmodule); JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name, jl_value_t *lb, jl_value_t *ub); @@ -1430,7 +1425,7 @@ JL_DLLEXPORT const char *jl_pathname_for_handle(void *handle); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); JL_DLLEXPORT void jl_set_sysimg_so(void *handle); -JL_DLLEXPORT ios_t *jl_create_system_image(void); +JL_DLLEXPORT ios_t *jl_create_system_image(void *); JL_DLLEXPORT void jl_save_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); @@ -1792,7 +1787,6 @@ typedef struct { const char *bindto; const char *outputbc; const char *outputunoptbc; - const char *outputjitbc; const char *outputo; const char *outputji; int8_t incremental; @@ -1899,8 +1893,6 @@ typedef struct { // codegen interface ---------------------------------------------------------- typedef struct { - int cached; // can the compiler use/populate the compilation cache? - int track_allocations; // can we track allocations? int code_coverage; // can we measure coverage? int static_alloc; // is the compiler allowed to allocate statically? diff --git a/src/julia_internal.h b/src/julia_internal.h index 1abeda26c327b..4206607ace88b 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -322,13 +322,11 @@ jl_svec_t *jl_perm_symsvec(size_t n, ...); jl_value_t *jl_gc_realloc_string(jl_value_t *s, size_t sz); jl_code_info_t *jl_type_infer(jl_method_instance_t **li JL_ROOTS_TEMPORARILY, size_t world, int force); -jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t *li, const char *F, size_t world); -jl_llvm_functions_t jl_compile_linfo( - jl_method_instance_t **pli, - jl_code_info_t *src JL_MAYBE_UNROOTED, - size_t world, - const jl_cgparams_t *params); -jl_llvm_functions_t jl_compile_for_dispatch(jl_method_instance_t **li, size_t world); +jl_generic_fptr_t jl_generate_fptr(jl_method_instance_t **pli, size_t world); +jl_generic_fptr_t jl_generate_fptr_for_unspecialized(jl_method_instance_t *unspec); +void jl_generate_ir(jl_method_instance_t *li, jl_code_info_t *src, size_t world); +void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params); +jl_generic_fptr_t jl_compile_for_dispatch(jl_method_instance_t **li, size_t world); JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types); jl_value_t *jl_interpret_call(jl_method_instance_t *lam, jl_value_t **args, uint32_t nargs); jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *lam); @@ -336,57 +334,30 @@ int jl_code_requires_compiler(jl_code_info_t *src); jl_code_info_t *jl_new_code_info_from_ast(jl_expr_t *ast); JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); -STATIC_INLINE jl_value_t *jl_compile_method_internal(jl_generic_fptr_t *fptr, - jl_method_instance_t *meth) +STATIC_INLINE jl_generic_fptr_t jl_compile_method_internal(jl_method_instance_t *meth) { - if (meth->jlcall_api == JL_API_CONST) - return jl_assume(meth->inferred_const); - fptr->fptr = meth->fptr; - fptr->jlcall_api = meth->jlcall_api; - if (__unlikely(fptr->fptr == NULL || fptr->jlcall_api == 0)) { + jl_generic_fptr_t fptr; + fptr.fptr = meth->fptr; + fptr.jlcall_api = meth->jlcall_api; + if (__unlikely(fptr.fptr == NULL || fptr.jlcall_api == JL_API_NOT_SET)) { size_t world = jl_get_ptls_states()->world_age; - // first see if it likely needs to be compiled - const char *F = meth->functionObjectsDecls.functionObject; - if (!F) // ask codegen to try to turn it into llvm code - F = jl_compile_for_dispatch(&meth, world).functionObject; - if (meth->jlcall_api == JL_API_CONST) - return jl_assume(meth->inferred_const); - // if it hasn't been inferred, try using the unspecialized meth cache instead - if (!meth->inferred) { - fptr->fptr = meth->unspecialized_ducttape; - fptr->jlcall_api = JL_API_GENERIC; - if (!fptr->fptr) { - if (jl_is_method(meth->def.method) && meth->def.method->unspecialized) { - fptr->fptr = meth->def.method->unspecialized->fptr; - fptr->jlcall_api = meth->def.method->unspecialized->jlcall_api; - if (fptr->jlcall_api == 2) { - return jl_assume(meth->def.method->unspecialized->inferred_const); - } - } - } - } - if (!fptr->fptr || fptr->jlcall_api == 0) { - // ask codegen to make the fptr - *fptr = jl_generate_fptr(meth, F, world); - if (fptr->jlcall_api == JL_API_CONST) - return jl_assume(meth->inferred_const); - } + fptr = jl_compile_for_dispatch(&meth, world); } - return NULL; + return fptr; } -STATIC_INLINE jl_value_t *jl_call_fptr_internal(const jl_generic_fptr_t *fptr, +STATIC_INLINE jl_value_t *jl_call_fptr_internal(jl_generic_fptr_t fptr, jl_method_instance_t *meth, jl_value_t **args, uint32_t nargs) { - if (fptr->jlcall_api == JL_API_GENERIC) - return fptr->fptr1(args[0], &args[1], nargs-1); - else if (fptr->jlcall_api == JL_API_CONST) - return meth->inferred; - else if (fptr->jlcall_api == JL_API_WITH_PARAMETERS) - return fptr->fptr3(meth->sparam_vals, args[0], &args[1], nargs-1); - else if (fptr->jlcall_api == JL_API_INTERPRETED) - return fptr->fptr4(meth, &args[0], nargs, meth->sparam_vals); + if (fptr.jlcall_api == JL_API_GENERIC) + return fptr.fptr1(args[0], &args[1], nargs - 1); + if (fptr.jlcall_api == JL_API_CONST) + return fptr.constant_2; + else if (fptr.jlcall_api == JL_API_WITH_PARAMETERS) + return fptr.fptr3(meth->sparam_vals, args[0], &args[1], nargs - 1); + else if (fptr.jlcall_api == JL_API_INTERPRETED) + return fptr.fptr4(meth, &args[0], nargs, meth->sparam_vals); else abort(); } @@ -394,12 +365,7 @@ STATIC_INLINE jl_value_t *jl_call_fptr_internal(const jl_generic_fptr_t *fptr, // invoke (compiling if necessary) the jlcall function pointer for a method STATIC_INLINE jl_value_t *jl_call_method_internal(jl_method_instance_t *meth, jl_value_t **args, uint32_t nargs) { - jl_generic_fptr_t fptr; - jl_value_t *v = jl_compile_method_internal(&fptr, meth); - if (v) - return v; - (void)jl_assume(fptr.jlcall_api != JL_API_CONST); - return jl_call_fptr_internal(&fptr, meth, args, nargs); + return jl_call_fptr_internal(jl_compile_method_internal(meth), meth, args, nargs); } jl_value_t *jl_argtype_with_function(jl_function_t *f, jl_value_t *types); @@ -549,7 +515,7 @@ void jl_init_types(void); void jl_init_box_caches(void); void jl_init_frontend(void); void jl_init_primitives(void); -void *jl_init_llvm(void); +void jl_init_llvm(void); void jl_init_codegen(void); void jl_init_intrinsic_functions(void); void jl_init_intrinsic_properties(void); @@ -632,8 +598,13 @@ static inline void jl_set_gc_and_wait(void) JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, int raw_mc, const char* asm_variant); -void jl_dump_native(const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len); -int32_t jl_get_llvm_gv(jl_value_t *p); +void *jl_create_native(jl_array_t *methods); +void jl_dump_native(void *native_code, + const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, + const char *sysimg_data, size_t sysimg_len); +int32_t jl_get_llvm_gv(void *native_code, jl_value_t *p); +void jl_get_function_id(void *native_code, jl_method_instance_t *linfo, + uint8_t *api, uint32_t *func_idx, uint32_t *specfunc_idx); int32_t jl_assign_functionID(const char *fname); int32_t jl_jlcall_api(const char *fname); // the first argument to jl_idtable_rehash is used to return a value @@ -652,7 +623,6 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller); uint32_t jl_module_next_counter(jl_module_t *m); -void jl_fptr_to_llvm(jl_fptr_t fptr, jl_method_instance_t *lam, int specsig); jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs); int jl_has_meta(jl_array_t *body, jl_sym_t *sym); diff --git a/src/method.c b/src/method.c index 2ad7cdf7f8092..5176fdecc47b4 100644 --- a/src/method.c +++ b/src/method.c @@ -236,11 +236,9 @@ JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void) li->sparam_vals = jl_emptysvec; li->backedges = NULL; li->fptr = NULL; - li->unspecialized_ducttape = NULL; + li->fptr_specsig = NULL; li->jlcall_api = 0; li->compile_traced = 0; - li->functionObjectsDecls.functionObject = NULL; - li->functionObjectsDecls.specFunctionObject = NULL; li->specTypes = NULL; li->inInference = 0; li->def.value = NULL; diff --git a/src/precompile.c b/src/precompile.c index 24293fb3ab97f..b60586a7e882b 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -20,28 +20,23 @@ JL_DLLEXPORT int jl_generating_output(void) return jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputji; } -void jl_precompile(int all); +void *jl_precompile(int all); void jl_write_compiler_output(void) { if (!jl_generating_output()) { - if (jl_options.outputjitbc) - jl_dump_native(NULL, jl_options.outputjitbc, NULL, NULL, 0); return; } + void *native_code = NULL; if (!jl_options.incremental) - jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); + native_code = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); if (!jl_module_init_order) { jl_printf(JL_STDERR, "WARNING: --output requested, but no modules defined during run\n"); return; } - if (jl_options.outputjitbc) { - jl_printf(JL_STDERR, "WARNING: --output-jit-bc is meaningless with options for dumping sysimage data\n"); - } - jl_array_t *worklist = jl_module_init_order; JL_GC_PUSH1(&worklist); jl_module_init_order = jl_alloc_vec_any(0); @@ -65,7 +60,7 @@ void jl_write_compiler_output(void) else { ios_t *s = NULL; if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc) - s = jl_create_system_image(); + s = jl_create_system_image(native_code); if (jl_options.outputji) { if (s == NULL) { @@ -81,7 +76,8 @@ void jl_write_compiler_output(void) } if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc) - jl_dump_native(jl_options.outputbc, + jl_dump_native(native_code, + jl_options.outputbc, jl_options.outputunoptbc, jl_options.outputo, (const char*)s->buf, (size_t)s->size); @@ -229,8 +225,7 @@ static void _compile_all_deq(jl_array_t *found) int found_i, found_l = jl_array_len(found); jl_printf(JL_STDERR, "found %d uncompiled methods for compile-all\n", (int)found_l); jl_method_instance_t *linfo = NULL; - jl_value_t *src = NULL; - JL_GC_PUSH2(&linfo, &src); + JL_GC_PUSH1(&linfo); for (found_i = 0; found_i < found_l; found_i++) { if (found_i % (1 + found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files jl_printf(JL_STDERR, " %d / %d\r", found_i + 1, found_l); @@ -247,7 +242,6 @@ static void _compile_all_deq(jl_array_t *found) if (linfo->jlcall_api == JL_API_CONST) continue; - src = m->source; // TODO: the `unspecialized` field is not yet world-aware, so we can't store // an inference result there. //src = jl_type_infer(&linfo, jl_world_counter, 1); @@ -259,8 +253,7 @@ static void _compile_all_deq(jl_array_t *found) // first try to create leaf signatures from the signature declaration and compile those _compile_all_union((jl_value_t*)ml->sig); // then also compile the generic fallback - jl_compile_linfo(&linfo, (jl_code_info_t*)src, jl_world_counter, &jl_default_cgparams); - assert(linfo->functionObjectsDecls.functionObject != NULL); + (void)jl_generate_fptr_for_unspecialized(linfo); } JL_GC_POP(); jl_printf(JL_STDERR, "\n"); @@ -273,8 +266,7 @@ static int compile_all_enq__(jl_typemap_entry_t *ml, void *env) jl_method_t *m = ml->func.method; if (m->source && (!m->unspecialized || - (m->unspecialized->functionObjectsDecls.functionObject == NULL && - m->unspecialized->jlcall_api != JL_API_CONST && + (m->unspecialized->jlcall_api != JL_API_CONST && m->unspecialized->fptr == NULL))) { // found a lambda that still needs to be compiled jl_array_ptr_1d_push(found, (jl_value_t*)ml); @@ -309,10 +301,10 @@ static void jl_compile_all_defs(void) static int precompile_enq_specialization_(jl_typemap_entry_t *l, void *closure) { - if (jl_is_method_instance(l->func.value) && - l->func.linfo->functionObjectsDecls.functionObject == NULL && - l->func.linfo->jlcall_api != JL_API_CONST) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)l->sig); + jl_method_instance_t *mi = l->func.linfo; + if (mi && jl_is_method_instance(mi) && mi->jlcall_api != JL_API_CONST) { + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + } return 1; } @@ -327,25 +319,34 @@ static void precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) jl_typemap_visitor(mt->defs, precompile_enq_all_specializations__, env); } -static void jl_compile_specializations(void) +void *jl_precompile(int all) { + if (all) + jl_compile_all_defs(); // this "found" array will contain function // type signatures that were inferred but haven't been compiled jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); + jl_array_t *m2 = NULL; + JL_GC_PUSH2(&m, &m2); jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); - size_t i, l; - for (i = 0, l = jl_array_len(m); i < l; i++) { - jl_compile_hint((jl_tupletype_t*)jl_array_ptr_ref(m, i)); + m2 = jl_alloc_vec_any(0); + for (size_t i = 0; i < jl_array_len(m); i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(m, i); + if (!jl_is_cacheable_sig((jl_tupletype_t*)mi->specTypes, (jl_tupletype_t*)mi->def.method->sig, mi->def.method)) { + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_world_counter); + } + else if (mi->max_world != ~(size_t)0) { + if (mi->min_world <= jl_typeinf_world && jl_typeinf_world <= mi->max_world) + jl_array_ptr_1d_push(m2, (jl_value_t*)mi); + mi = jl_specializations_get_linfo(mi->def.method, mi->specTypes, mi->sparam_vals, jl_world_counter); + } + if (mi) + jl_array_ptr_1d_push(m2, (jl_value_t*)mi); } + m = NULL; + void *native_code = jl_create_native(m2); JL_GC_POP(); -} - -void jl_precompile(int all) -{ - if (all) - jl_compile_all_defs(); - jl_compile_specializations(); + return native_code; } #ifdef __cplusplus diff --git a/src/staticdata.c b/src/staticdata.c index 2e76be1f2f9dc..ce2472aabdee5 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -54,6 +54,8 @@ jl_array_t *jl_module_init_order; // hash of definitions for predefined function pointers static htable_t fptr_to_id; +void *native_functions; + // array of definitions for the predefined function pointers // (reverse of fptr_to_id) static const jl_fptr_t id_to_fptrs[] = { @@ -66,16 +68,6 @@ static const jl_fptr_t id_to_fptrs[] = { jl_f_applicable, jl_f_invoke, jl_f_sizeof, jl_f__expr, NULL }; -typedef enum _DUMP_MODES { - // not in the serializer at all, or - // something is seriously wrong - MODE_INVALID = 0, - - // jl_restore_system_image - // restoring an entire system image from disk - MODE_SYSTEM_IMAGE, -} DUMP_MODES; - typedef struct { ios_t *s; ios_t *const_data; @@ -85,7 +77,6 @@ typedef struct { ios_t *fptr_record; arraylist_t relocs_list; arraylist_t gctags_list; - DUMP_MODES mode; jl_ptls_t ptls; } jl_serializer_state; @@ -199,20 +190,6 @@ static uintptr_t jl_fptr_id(void *fptr) return *(uintptr_t*)pbp; } -int32_t jl_jlcall_api(const char *fname) -{ - // give the function an index in the constant lookup table - if (fname == NULL) - return 0; - if (!strncmp(fname, "japi3_", 6)) // jlcall abi 3 from JIT - return JL_API_WITH_PARAMETERS; - assert(!strncmp(fname, "japi1_", 6) || // jlcall abi 1 from JIT - !strncmp(fname, "jsys1_", 6) || // jlcall abi 1 from sysimg - !strncmp(fname, "jlcall_", 7) || // jlcall abi 1 from JIT wrapping a specsig method - !strncmp(fname, "jlsysw_", 7)); // jlcall abi 1 from sysimg wrapping a specsig method - return JL_API_GENERIC; -} - #define jl_serialize_value(s, v) jl_serialize_value_(s,(jl_value_t*)(v)) static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v); @@ -422,7 +399,8 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t write_gctaggedfield(s, (uintptr_t)BindingRef << RELOC_TAG_OFFSET); tot += sizeof(void*); size_t binding_reloc_offset = ios_pos(s->s); - record_gvar(s, jl_get_llvm_gv((jl_value_t*)b), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset); + record_gvar(s, jl_get_llvm_gv(native_functions, (jl_value_t*)b), + ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset); write_pointerfield(s, (jl_value_t*)b->name); write_pointerfield(s, b->value); write_pointerfield(s, b->globalref); @@ -555,7 +533,7 @@ static void jl_write_values(jl_serializer_state *s) size_t reloc_offset = ios_pos(s->s); assert(item < layout_table.len && layout_table.items[item] == NULL); layout_table.items[item] = (void*)reloc_offset; - record_gvar(s, jl_get_llvm_gv(v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset); + record_gvar(s, jl_get_llvm_gv(native_functions, v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset); // write data if (jl_is_cpointer(v)) { @@ -660,10 +638,12 @@ static void jl_write_values(jl_serializer_state *s) jl_method_instance_t *m = (jl_method_instance_t*)v; jl_method_instance_t *newm = (jl_method_instance_t*)&s->s->buf[reloc_offset]; newm->fptr = NULL; - newm->unspecialized_ducttape = NULL; + newm->fptr_specsig = NULL; if (jl_is_method(m->def.method)) { uintptr_t fptr_id = jl_fptr_id((void*)(uintptr_t)m->fptr); if (m->jlcall_api == JL_API_CONST) { + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_method_instance_t, fptr))); // relocation location + arraylist_push(&s->relocs_list, (void*)backref_id(s, m->inferred_const)); // relocation target } else if (fptr_id >= 2) { //write_int8(s->s, -li->jlcall_api); @@ -672,13 +652,12 @@ static void jl_write_values(jl_serializer_state *s) arraylist_push(&reinit_list, (void*)item); arraylist_push(&reinit_list, (void*)6); } - else if (m->functionObjectsDecls.functionObject) { - int jlcall_api = jl_jlcall_api(m->functionObjectsDecls.functionObject); - assert(jlcall_api); - newm->jlcall_api = jlcall_api; + else { // save functionObject pointers - int cfunc = jl_assign_functionID(m->functionObjectsDecls.specFunctionObject); - int func = jl_assign_functionID(m->functionObjectsDecls.functionObject); + uint8_t api = JL_API_NOT_SET; + uint32_t cfunc = 0; + uint32_t func = 0; + jl_get_function_id(native_functions, m, &api, &func, &cfunc); assert(reloc_offset < INT32_MAX); if (cfunc != 0) { ios_ensureroom(s->fptr_record, cfunc * sizeof(void*)); @@ -696,13 +675,9 @@ static void jl_write_values(jl_serializer_state *s) write_padding(s->fptr_record, 4); #endif } - } - else { - newm->jlcall_api = 0; + newm->jlcall_api = api; } } - newm->functionObjectsDecls.functionObject = NULL; - newm->functionObjectsDecls.specFunctionObject = NULL; } else if (jl_is_datatype(v)) { jl_datatype_t *dt = (jl_datatype_t*)v; @@ -753,7 +728,7 @@ static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v) { // since symbols are static, they might not have had a // reference anywhere in the code image other than here - int32_t gv = jl_get_llvm_gv((jl_value_t*)v); + int32_t gv = jl_get_llvm_gv(native_functions, (jl_value_t*)v); if (gv != 0) { uintptr_t item = backref_id(s, v); assert(item >> RELOC_TAG_OFFSET == SymbolRef); @@ -961,7 +936,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s) } jl_method_instance_t *li = (jl_method_instance_t*)(base + offset); if (fvars.base == NULL) { - li->jlcall_api = 0; + li->jlcall_api = JL_API_NOT_SET; } else { uintptr_t base = (uintptr_t)fvars.base; @@ -976,7 +951,13 @@ static void jl_update_all_fptrs(jl_serializer_state *s) offset = fvars.clone_offsets[clone_idx]; break; } - jl_fptr_to_llvm((jl_fptr_t)(base + offset), li, cfunc); + jl_fptr_t fptr = (jl_fptr_t)(base + offset); + if (cfunc) { + li->fptr_specsig = fptr; + } + else { + li->fptr = fptr; + } } } } @@ -1150,11 +1131,12 @@ static void jl_prune_type_cache(jl_svec_t *cache) jl_value_t *ti = jl_svecref(cache, i); if (ti == NULL) break; - if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(ti) != 0) + if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(native_functions, ti) != 0) jl_svecset(cache, ins++, ti); else if (jl_is_datatype(ti)) { jl_value_t *singleton = ((jl_datatype_t*)ti)->instance; - if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND || jl_get_llvm_gv(singleton) != 0)) + if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND || + jl_get_llvm_gv(native_functions, singleton) != 0)) jl_svecset(cache, ins++, ti); } } @@ -1193,7 +1175,6 @@ static void jl_save_system_image_to_stream(ios_t *f) s.relocs = &relocs; s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; - s.mode = MODE_SYSTEM_IMAGE; s.ptls = jl_get_ptls_states(); arraylist_new(&s.relocs_list, 0); arraylist_new(&s.gctags_list, 0); @@ -1243,7 +1224,7 @@ static void jl_save_system_image_to_stream(ios_t *f) uintptr_t i; for (i = 0; i < deser_tag.len; i++) { jl_value_t *v = (jl_value_t*)deser_tag.items[i]; - record_gvar(&s, jl_get_llvm_gv(v), ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i); + record_gvar(&s, jl_get_llvm_gv(native_functions, v), ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i); } } @@ -1311,10 +1292,11 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_gc_enable(en); } -JL_DLLEXPORT ios_t *jl_create_system_image(void) +JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data) { ios_t *f = (ios_t*)malloc(sizeof(ios_t)); ios_mem(f, 0); + native_functions = _native_data; jl_save_system_image_to_stream(f); return f; } @@ -1376,7 +1358,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) s.relocs = &relocs; s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; - s.mode = MODE_SYSTEM_IMAGE; s.ptls = jl_get_ptls_states(); arraylist_new(&s.relocs_list, 0); arraylist_new(&s.gctags_list, 0); diff --git a/src/threading.c b/src/threading.c index 472b01f750d73..abdc5faa92b74 100644 --- a/src/threading.c +++ b/src/threading.c @@ -309,7 +309,7 @@ static jl_value_t *ti_run_fun(const jl_generic_fptr_t *fptr, jl_method_instance_ jl_ptls_t ptls = jl_get_ptls_states(); JL_TRY { (void)jl_assume(fptr->jlcall_api != JL_API_CONST); - jl_call_fptr_internal(fptr, mfunc, args, nargs); + jl_call_fptr_internal(*fptr, mfunc, args, nargs); } JL_CATCH { // Lock this output since we know it'll likely happen on multiple threads @@ -689,8 +689,9 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_value_t *_args) threadwork.command = TI_THREADWORK_RUN; threadwork.mfunc = jl_lookup_generic(args, nargs, jl_int32hash_fast(jl_return_address()), ptls->world_age); - // Ignore constant return value for now. - if (jl_compile_method_internal(&threadwork.fptr, threadwork.mfunc)) + threadwork.fptr = jl_compile_method_internal(threadwork.mfunc); + if (threadwork.fptr.jlcall_api == JL_API_CONST) + // Ignore constant return value for now. return jl_nothing; threadwork.args = args; threadwork.nargs = nargs; @@ -804,8 +805,9 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_value_t *_args) jl_method_instance_t *mfunc = jl_lookup_generic(args, nargs, jl_int32hash_fast(jl_return_address()), jl_get_ptls_states()->world_age); - jl_generic_fptr_t fptr; - if (jl_compile_method_internal(&fptr, mfunc)) + jl_generic_fptr_t fptr = jl_compile_method_internal(mfunc); + if (fptr.jlcall_api == JL_API_CONST) + // Ignore constant return value for now. return jl_nothing; return ti_run_fun(&fptr, mfunc, args, nargs); } diff --git a/stdlib/InteractiveUtils/src/codeview.jl b/stdlib/InteractiveUtils/src/codeview.jl index 2b42d010161e7..2ad8bb3c9b753 100644 --- a/stdlib/InteractiveUtils/src/codeview.jl +++ b/stdlib/InteractiveUtils/src/codeview.jl @@ -79,34 +79,33 @@ function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrappe meth = Base.func_for_method_checked(meth, ti) linfo = ccall(:jl_specializations_get_linfo, Ref{Core.MethodInstance}, (Any, Any, Any, UInt), meth, ti, env, world) # get the code for it - return _dump_function_linfo(linfo, world, native, wrapper, strip_ir_metadata, dump_module, syntax, optimize, params) -end - -function _dump_function_linfo(linfo::Core.MethodInstance, world::UInt, native::Bool, wrapper::Bool, - strip_ir_metadata::Bool, dump_module::Bool, syntax::Symbol=:att, - optimize::Bool=true, params::CodegenParams=CodegenParams()) - if syntax != :att && syntax != :intel - throw(ArgumentError("'syntax' must be either :intel or :att")) - end if native - llvmf = ccall(:jl_get_llvmf_decl, Ptr{Cvoid}, (Any, UInt, Bool, CodegenParams), linfo, world, wrapper, params) + str = _dump_function_linfo_native(linfo, world, wrapper, syntax) else - llvmf = ccall(:jl_get_llvmf_defn, Ptr{Cvoid}, (Any, UInt, Bool, Bool, CodegenParams), linfo, world, wrapper, optimize, params) - end - if llvmf == C_NULL - error("could not compile the specified method") + str = _dump_function_linfo_llvm(linfo, world, wrapper, strip_ir_metadata, dump_module, optimize, params) end + # TODO: use jl_is_cacheable_sig instead of isdispatchtuple + isdispatchtuple(linfo.specTypes) || (str = "; WARNING: This code may not match what actually runs.\n" * str) + return str +end - if native - str = ccall(:jl_dump_function_asm, Ref{String}, - (Ptr{Cvoid}, Cint, Ptr{UInt8}), llvmf, 0, syntax) - else - str = ccall(:jl_dump_function_ir, Ref{String}, - (Ptr{Cvoid}, Bool, Bool), llvmf, strip_ir_metadata, dump_module) +function _dump_function_linfo_native(linfo::Core.MethodInstance, world::UInt, wrapper::Bool, syntax::Symbol=:att) + if syntax != :att && syntax != :intel + throw(ArgumentError("'syntax' must be either :intel or :att")) end + str = ccall(:jl_dump_method_asm, Ref{String}, + (Any, UInt, Cint, Bool, Ptr{UInt8}), linfo, world, 0, wrapper, syntax) + return str +end - # TODO: use jl_is_cacheable_sig instead of isdispatchtuple - isdispatchtuple(linfo.specTypes) || (str = "; WARNING: This code may not match what actually runs.\n" * str) +function _dump_function_linfo_llvm( + linfo::Core.MethodInstance, world::UInt, wrapper::Bool, + strip_ir_metadata::Bool, dump_module::Bool, + optimize::Bool=true, params::CodegenParams=CodegenParams()) + llvmf = ccall(:jl_get_llvmf_defn, Ptr{Cvoid}, (Any, UInt, Bool, Bool, CodegenParams), linfo, world, wrapper, optimize, params) + llvmf == C_NULL && error("could not compile the specified method") + str = ccall(:jl_dump_function_ir, Ref{String}, + (Ptr{Cvoid}, Bool, Bool), llvmf, strip_ir_metadata, dump_module) return str end @@ -132,5 +131,6 @@ Switch assembly syntax using `syntax` symbol parameter set to `:att` for AT&T sy """ code_native(io::IO, @nospecialize(f), @nospecialize(types=Tuple); syntax::Symbol = :att) = print(io, _dump_function(f, types, true, false, false, false, syntax)) -code_native(@nospecialize(f), @nospecialize(types=Tuple); syntax::Symbol = :att) = code_native(STDOUT, f, types, syntax = syntax) +code_native(@nospecialize(f), @nospecialize(types=Tuple); syntax::Symbol = :att) = + code_native(STDOUT, f, types, syntax = syntax) code_native(::IO, ::Any, ::Symbol) = error("illegal code_native call") # resolve ambiguous call diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index d13cb94d37013..a643a2680babd 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -240,13 +240,25 @@ end @which get_A18434()(1, y=2) @test counter18434 == 2 -let _true = Ref(true), f, g, h - @noinline f() = ccall((:time, "error_library_doesnt_exist\0"), Cvoid, ()) # some expression that throws an error in codegen - @noinline g() = _true[] ? 0 : h() - @noinline h() = (g(); f()) - @test_throws ErrorException @code_native h() # due to a failure to compile f() - @test g() == 0 +# manually generate a broken function, which will break codegen +# and make sure Julia doesn't crash +@eval @noinline f_broken_code() = 0 +let m = which(f_broken_code, ()) + let src = Base.uncompressed_ast(m) + src.code = Any[ + Expr(:meta, :noinline) + Expr(:return, Expr(:invalid)) + ] + m.source = src + end end +_true = true +# and show that we can still work around it +@noinline g_broken_code() = _true ? 0 : h_broken_code() +@noinline h_broken_code() = (g_broken_code(); f_broken_code()) +@test contains(sprint(code_native, h_broken_code, ()), "h_broken_code") +@test g_broken_code() == 0 + module ReflectionTest using Test, Random, InteractiveUtils diff --git a/test/core.jl b/test/core.jl index 64b676f3bd44a..d2bbfdea9a773 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4736,9 +4736,9 @@ end cfunction(f18054, Cint, Tuple{}) # issue #18986: the ccall optimization of cfunction leaves JL_TRY stack in bad state -dummy18996() = return nothing +dummy18986() = return nothing function main18986() - cfunction(dummy18986, Cvoid, ()) + cfunction(dummy18986, Cvoid, Tuple{}) ccall((:dummy2, "this_is_a_nonexisting_library"), Cvoid, ()) end @test_throws ErrorException main18986() diff --git a/test/reflection.jl b/test/reflection.jl index 9993db250ecbd..45802d9c83cd2 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -263,22 +263,6 @@ end end @test functionloc(f14346)[2] == @__LINE__() - 4 -# test jl_get_llvm_fptr. We test functions both in and definitely not in the system image -definitely_not_in_sysimg() = nothing -for (f, t) in Any[(definitely_not_in_sysimg, Tuple{}), - (Base.:+, Tuple{Int, Int})] - meth = which(f, t) - tt = Tuple{typeof(f), t.parameters...} - (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), tt, meth.sig)::Core.SimpleVector - @test ti === tt # intersection should be a subtype - world = typemax(UInt) - linfo = ccall(:jl_specializations_get_linfo, Ref{Core.MethodInstance}, (Any, Any, Any, UInt), meth, tt, env, world) - params = Base.CodegenParams() - llvmf = ccall(:jl_get_llvmf_decl, Ptr{Cvoid}, (Any, UInt, Bool, Base.CodegenParams), linfo::Core.MethodInstance, world, true, params) - @test llvmf != C_NULL - @test ccall(:jl_get_llvm_fptr, Ptr{Cvoid}, (Ptr{Cvoid},), llvmf) != C_NULL -end - # issue #15714 # show variable names for slots and suppress spurious type warnings function f15714(array_var15714) diff --git a/test/staged.jl b/test/staged.jl index 8d1cb76be7253..1336134eaf977 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -191,7 +191,9 @@ let gf_err2 return nothing end @test_throws ErrorException gf_err2(code_typed) - @test gf_err_ref[] == 4 + @test_throws ErrorException gf_err2(code_llvm) + @test_throws ErrorException gf_err2(code_native) + @test gf_err_ref[] == 12 @test gf_err2(code_lowered) === nothing end