From 2a65a9097a8dd0870a71ef3bb0caf968bba8c4e7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Mar 2016 02:42:59 -0400 Subject: [PATCH] simplify interaction between new JITs and codegen this treats the new JITs (MCJIT, ORCJIT) much like the old JIT, but using Module as the atomic unit instead of Function this gets codegen closer to being separable (cachable) at the module level, with the remerging happening in the JIT after object file emission TODO: memory optimization of strings in the JIT needs to be reimplemented fix #15533 --- src/Makefile | 4 +- src/ccall.cpp | 141 ++----- src/cgutils.cpp | 762 ++++--------------------------------- src/codegen.cpp | 843 +++++++++++++---------------------------- src/codegen_internal.h | 14 +- src/debuginfo.cpp | 4 + src/dump.c | 2 +- src/gf.c | 14 +- src/intrinsics.cpp | 81 ++-- src/jitlayers.cpp | 591 +++++++++++++++++++++++++++-- src/julia.h | 1 + src/julia_internal.h | 6 +- src/toplevel.c | 2 +- test/llvmcall.jl | 19 +- 14 files changed, 1028 insertions(+), 1456 deletions(-) diff --git a/src/Makefile b/src/Makefile index ae46d015acc6e..5f7e50c34b190 100644 --- a/src/Makefile +++ b/src/Makefile @@ -117,8 +117,10 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ # additional dependency links $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp intrinsics.h cgutils.cpp ccall.cpp jitlayers.cpp abi_*.cpp) +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp jitlayers.cpp intrinsics.h codegen_internal.h cgutils.cpp ccall.cpp abi_*.cpp) $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/codegen_internal.h +$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/codegen_internal.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-debug.c $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0bf253f03d839..833e54e562030 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -4,26 +4,8 @@ // --- the ccall, cglobal, and llvm intrinsics --- -// keep track of llvmcall declarations -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static std::map llvmcallDecls; -#else -static std::set llvmcallDecls; -#endif - -static std::map libMapGV; -static std::map symMapGV; - -static Value *GetStringPtr(llvm::IRBuilder <> *builder, GlobalValue *GV, const Twine &Name) -{ - Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); - Value *Args[] = { zero, zero }; -#ifdef LLVM37 - return builder->CreateInBoundsGEP(GV->getValueType(), GV, Args, Name); -#else - return builder->CreateInBoundsGEP(GV, Args, Name); -#endif -} +static StringMap libMapGV; +static StringMap symMapGV; static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, const char *f_name, jl_codectx_t *ctx) { @@ -53,52 +35,47 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons libsym = jl_RTLD_DEFAULT_handle; } else { + std::string name = "ccalllib_"; + name += f_lib; runtime_lib = true; libptrgv = libMapGV[f_lib]; if (libptrgv == NULL) { - libptrgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pint8, - false, GlobalVariable::PrivateLinkage, - ConstantPointerNull::get((PointerType*)T_pint8), f_lib); - libMapGV[f_lib] = libptrgv; + libptrgv = new GlobalVariable(*jl_Module, T_pint8, + false, GlobalVariable::ExternalLinkage, + NULL, name); + libMapGV[f_lib] = global_proto(libptrgv); libsym = jl_get_library(f_lib); assert(libsym != NULL); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[libptrgv] = libsym; -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)) = libsym; -#endif + *(void**)jl_emit_and_add_to_shadow(libptrgv) = libsym; + } + else { + libptrgv = prepare_global(libptrgv); } } if (libsym == NULL) { -#ifdef USE_MCJIT - libsym = (void*)jl_llvm_to_jl_value[libptrgv]; -#else - libsym = *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)); -#endif + libsym = *(void**)jl_get_global(libptrgv); } - assert(libsym != NULL); GlobalVariable *llvmgv = symMapGV[f_name]; - Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); if (llvmgv == NULL) { // MCJIT forces this to have external linkage eventually, so we would clobber // the symbol of the actual function. - std::string name = f_name; - name = "ccall_" + name; - llvmgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pvoidfunc, - false, GlobalVariable::PrivateLinkage, - initnul, name); - symMapGV[f_name] = llvmgv; -#ifdef USE_MCJIT - jl_llvm_to_jl_value[llvmgv] = jl_dlsym_e(libsym, f_name); -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(llvmgv)) = jl_dlsym_e(libsym, f_name); -#endif + std::string name = "ccall_"; + name += f_name; + llvmgv = new GlobalVariable(*jl_Module, T_pvoidfunc, + false, GlobalVariable::ExternalLinkage, + NULL, name); + symMapGV[f_name] = global_proto(llvmgv); + *(void**)jl_emit_and_add_to_shadow(llvmgv) = jl_dlsym_e(libsym, f_name); + } + else { + llvmgv = prepare_global(llvmgv); } BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"), *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); + Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); builder.CreateCondBr(builder.CreateICmpNE(builder.CreateLoad(llvmgv), initnul), ccall_bb, dlsym_lookup); assert(ctx->f->getParent() != NULL); @@ -598,7 +575,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::stringstream name; name << (ctx->f->getName().str()) << "u" << i++; ir_name = name.str(); - if (builtins_module->getFunction(ir_name) == NULL) + if (jl_Module->getFunction(ir_name) == NULL) break; } @@ -615,9 +592,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string rstring; llvm::raw_string_ostream rtypename(rstring); rettype->print(rtypename); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) std::map localDecls; -#endif if (decl != NULL) { std::stringstream declarations(jl_string_data(decl)); @@ -625,33 +600,16 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c // parse string line by line std::string declstr; while (std::getline(declarations, declstr, '\n')) { - uint64_t declhash = memhash(declstr.c_str(), declstr.length()); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - auto it = llvmcallDecls.find(declhash); - if (it != llvmcallDecls.end()) { - prepare_call(it->second); - } - else { -#else - if (llvmcallDecls.count(declhash) == 0) { -#endif - // Find name of declaration by searching for '@' - std::string::size_type atpos = declstr.find('@') + 1; - // Find end of declaration by searching for '(' - std::string::size_type bracepos = declstr.find('(', atpos); - // Declaration name is the string between @ and ( - std::string declname = declstr.substr(atpos, bracepos - atpos); - - // Check if declaration already present in module - if(jl_Module->getNamedValue(declname) == NULL) { - ir_stream << "; Declarations\n" << declstr << "\n"; - } - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - localDecls[declhash] = declname; -#else - llvmcallDecls.insert(declhash); -#endif + // Find name of declaration by searching for '@' + std::string::size_type atpos = declstr.find('@') + 1; + // Find end of declaration by searching for '(' + std::string::size_type bracepos = declstr.find('(', atpos); + // Declaration name is the string between @ and ( + std::string declname = declstr.substr(atpos, bracepos - atpos); + + // Check if declaration already present in module + if(jl_Module->getNamedValue(declname) == NULL) { + ir_stream << "; Declarations\n" << declstr << "\n"; } } } @@ -662,9 +620,9 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string ir_string = ir_stream.str(); #ifdef LLVM36 Module *m = NULL; - bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*builtins_module,Err); + bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*jl_Module,Err); if (!failed) - m = builtins_module; + m = jl_Module; #else Module *m = ParseAssemblyString(ir_string.c_str(),jl_Module,Err,jl_LLVMContext); #endif @@ -675,12 +633,8 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_error(stream.str().c_str()); } f = m->getFunction(ir_name); + f->removeFromParent(); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto it : localDecls) { - llvmcallDecls[it.first] = cast(prepare_call(m->getNamedValue(it.second))); - } -#endif } else { assert(isPtr); @@ -692,13 +646,6 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c it != argtypes.end(); ++it, ++i) assert(*it == f->getFunctionType()->getParamType(i)); -#ifdef USE_MCJIT - if (f->getParent() != active_module) { - FunctionMover mover(active_module,f->getParent()); - f = mover.CloneFunction(f); - } -#endif - //f->dump(); #ifndef LLVM35 if (verifyFunction(*f,PrintMessageAction)) { @@ -1174,8 +1121,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) if (jl_is_tuple_type(fargt) && jl_is_leaf_type(fargt)) { frt = jl_tparam0(frt); JL_TRY { - Value *llvmf = prepare_call( - jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); + Value *llvmf = prepare_call(jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); // make sure to emit any side-effects that may have been part of the original expression emit_expr(args[4], ctx); emit_expr(args[6], ctx); @@ -1353,8 +1299,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } if (needStackRestore) { - stacksave = CallInst::Create(prepare_call(Intrinsic::getDeclaration(builtins_module, - Intrinsic::stacksave))); + stacksave = CallInst::Create(Intrinsic::getDeclaration(jl_Module, + Intrinsic::stacksave)); if (savespot) { #ifdef LLVM38 instList.insertAfter(savespot->getIterator(), (Instruction*)stacksave); @@ -1381,10 +1327,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) result = ret; if (needStackRestore) { assert(stacksave != NULL); - builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, - Intrinsic::stackrestore)), - stacksave); + builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::stackrestore), stacksave); } if (0) { // Enable this to turn on SSPREQ (-fstack-protector) on the function containing this ccall ctx->f->addFnAttr(Attribute::StackProtectReq); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 8d54ffc2a074c..bd0a68e68d88f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4,482 +4,96 @@ // utility procedures used in code generation -template // for GlobalObject's -static T *addComdat(T *G) -{ -#if defined(_OS_WINDOWS_) - if (imaging_mode && !G->isDeclaration()) { -#ifdef LLVM35 - // Add comdat information to make MSVC link.exe happy - Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); - jl_Comdat->setSelectionKind(Comdat::NoDuplicates); - G->setComdat(jl_Comdat); - // add __declspec(dllexport) to everything marked for export - if (G->getLinkage() == GlobalValue::ExternalLinkage) - G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); -#endif - } -#endif - return G; -} - static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) { load_or_store->setMetadata( llvm::LLVMContext::MD_tbaa, md ); return load_or_store; } -// Fixing up references to other modules for MCJIT -std::set pending_globals; static GlobalVariable *prepare_global(GlobalVariable *G) { - pending_globals.insert(G); - return G; + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(G->getName()); + if (!local) { + local = global_proto(G, M); + } + return cast(local); } static llvm::Value *prepare_call(llvm::Value *Callee) { - llvm::Function *F = dyn_cast(Callee); - if (!F) - return Callee; - pending_globals.insert(F); - return Callee; -} - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalValue *realize_pending_global(Module *M, GlobalValue *G, std::map &FixedGlobals) -{ - if (M == G->getParent() && M != builtins_module) { // can happen during bootstrap - //std::cout << "Skipping " << std::string(G->getName()) << " due to parentage" << std::endl; - return nullptr; - } - // If we come across a function that is still being constructed, - // this use needs to remain pending - if (!M || M == builtins_module) { - pending_globals.insert(G); - //std::cout << "Skipping " << std::string(G->getName()) << " due to construction" << std::endl; - return nullptr; - } - if (!FixedGlobals.count(M)) { - if (GlobalVariable *GV = dyn_cast(G)) { - GlobalVariable *NewGV = M->getGlobalVariable(GV->getName()); - if (!NewGV) { - NewGV = new GlobalVariable(*M, GV->getType()->getElementType(), - GV->isConstant(), GlobalVariable::ExternalLinkage, - NULL, GV->getName(), NULL, GV->getThreadLocalMode(), - GV->getType()->getAddressSpace()); - NewGV->setUnnamedAddr(GV->hasUnnamedAddr()); - // Move over initializer - if (GV->hasInitializer()) { - NewGV->setInitializer(GV->getInitializer()); - GV->setInitializer(nullptr); - } - } - FixedGlobals[M] = NewGV; + if (Function *F = dyn_cast(Callee)) { + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(Callee->getName()); + if (!local) { + local = function_proto(F, M); } - else { - Function *F = cast(G); - //std::cout << "Realizing " << std::string(F->getName()) << std::endl; - //if (!F->getParent()) { - //std::cout << "Skipping" << std::endl; - // return nullptr; - //} - Function *NewF = nullptr; - if (!F->isDeclaration() && F->getParent() == builtins_module) { - // It's a definition. Actually move the function and create a - // declaration in the original module - NewF = F; - F->removeFromParent(); - M->getFunctionList().push_back(F); - Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - active_module); - } - else { - assert(F); - NewF = M->getFunction(F->getName()); - if (!NewF) { - NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - M); - } - } - FixedGlobals[M] = NewF; - } - } - return FixedGlobals[M]; -} - -struct ExprChain { - ConstantExpr *Expr; - unsigned OpNo; - struct ExprChain *Next; -}; - -static void handleUse(Use &Use1, llvm::GlobalValue *G, - std::map &FixedGlobals, - struct ExprChain *Chain, struct ExprChain *ChainEnd) -{ - Instruction *User = dyn_cast(Use1.getUser()); - GlobalValue *GVUser = dyn_cast(Use1.getUser()); - if (!User && !GVUser) { - ConstantExpr *Expr = cast(Use1.getUser()); - Value::use_iterator UI2 = Expr->use_begin(), E2 = Expr->use_end(); - for (; UI2 != E2;) { - Use &Use2 = *UI2; - ++UI2; - struct ExprChain NextChain; - NextChain.Expr = Expr; - NextChain.OpNo = Use1.getOperandNo(); - NextChain.Next = nullptr; - if (ChainEnd) - ChainEnd->Next = &NextChain; - handleUse(Use2,G,FixedGlobals,Chain ? Chain : &NextChain,&NextChain); - } - return; - } - llvm::Module *M = nullptr; - if (User) { - Function *UsedInHere = User->getParent()->getParent(); - assert(UsedInHere); - M = UsedInHere->getParent(); - } else { - assert(GVUser); - M = GVUser->getParent(); - } - llvm::Constant *Replacement = realize_pending_global(M,G,FixedGlobals); - if (!Replacement) - return; - while (Chain) { - Replacement = Chain->Expr->getWithOperandReplaced(Chain->OpNo,Replacement); - Chain->Expr = cast(Replacement); - Chain = Chain->Next; - } - Use1.set(Replacement); -} - -// RAUW, but only for those users which live in a module, and create a module -// specific copy -static void realize_pending_globals() -{ - std::set local_pending_globals; - std::swap(local_pending_globals,pending_globals); - std::map FixedGlobals; - for (auto *G : local_pending_globals) { - Value::use_iterator UI = G->use_begin(), E = G->use_end(); - for (; UI != E;) - handleUse(*(UI++),G,FixedGlobals,nullptr,nullptr); - FixedGlobals.clear(); - } -} - -static void realize_cycle(jl_cyclectx_t *cyclectx) -{ - // These need to be resolved together - for (auto *F : cyclectx->functions) { - F->removeFromParent(); - active_module->getFunctionList().push_back(F); - } - for (auto *CU : cyclectx->CUs) { - NamedMDNode *NMD = active_module->getOrInsertNamedMetadata("llvm.dbg.cu"); - NMD->addOperand(CU); - } -} -#endif - -template -#ifdef LLVM35 -static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) -#else -static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) -#endif -{ - // cast through integer to avoid c++ pedantic warning about casting between - // data and code pointers - void *addr = (void*)(uintptr_t)_addr; -#ifdef LLVM34 - StringRef name = gv->getName(); -#ifdef _OS_WINDOWS_ - std::string imp_name; -#endif -#endif - -#ifdef _OS_WINDOWS_ - // setting JL_DLLEXPORT correctly only matters when building a binary - if (dllimport && imaging_mode) { - assert(gv->getLinkage() == GlobalValue::ExternalLinkage); -#ifdef LLVM35 - // add the __declspec(dllimport) attribute - gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); - // this will cause llvm to rename it, so we do the same - imp_name = Twine("__imp_", name).str(); - name = StringRef(imp_name); -#else - gv->setLinkage(GlobalValue::DLLImportLinkage); -#endif -#if defined(_P64) || defined(LLVM35) - // __imp_ variables are indirection pointers, so use malloc to simulate that - void **imp_addr = (void**)malloc(sizeof(void**)); - *imp_addr = addr; - addr = (void*)imp_addr; -#endif + return local; } -#endif // _OS_WINDOWS_ - -#ifdef USE_ORCJIT - addComdat(gv); - jl_ExecutionEngine->addGlobalMapping(name, addr); -#elif defined(USE_MCJIT) - addComdat(gv); - sys::DynamicLibrary::AddSymbol(name, addr); -#else // USE_MCJIT - jl_ExecutionEngine->addGlobalMapping(gv, addr); -#endif // USE_MCJIT + return Callee; } // --- string constants --- -static std::map stringConstants; - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalVariable *global_proto(GlobalVariable *G) -{ - GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), - G->isConstant(), GlobalVariable::ExternalLinkage, - NULL, G->getName(), G->getThreadLocalMode()); - return proto; -} -#else -static GlobalVariable *global_proto(GlobalVariable *G) -{ - return G; -} -#endif - -static GlobalVariable *stringConst(const std::string &txt) +static StringMap stringConstants; +static GlobalValue *stringConst(const std::string &txt) { - GlobalVariable *gv = stringConstants[txt]; static int strno = 0; - // in inference, we can not share string constants between - // modules as there might be multiple compiles on the stack - // with calls in between them. - if (gv == NULL) { - std::stringstream ssno; - std::string vname; - ssno << strno; - vname += "_j_str"; - vname += ssno.str(); - gv = new GlobalVariable(*active_module, - ArrayType::get(T_int8, txt.length()+1), - true, - imaging_mode ? GlobalVariable::PrivateLinkage : GlobalVariable::ExternalLinkage, - ConstantDataArray::get(getGlobalContext(), - ArrayRef( - (const unsigned char*)txt.c_str(), - txt.length()+1)), - vname); - gv->setUnnamedAddr(true); - gv = imaging_mode ? gv : prepare_global(global_proto(gv)); - stringConstants[txt] = gv; - strno++; - } - else { - prepare_global(gv); - } + std::stringstream ssno; + ssno << "_j_str" << strno++; + GlobalVariable *gv = new GlobalVariable(*jl_builderModule, + ArrayType::get(T_int8, txt.length()+1), + true, + GlobalVariable::PrivateLinkage, + ConstantDataArray::get(getGlobalContext(), + ArrayRef( + (const unsigned char*)txt.c_str(), + txt.length()+1)), + ssno.str()); + gv->setUnnamedAddr(true); return gv; -} - -// --- Shadow module handling --- - -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing -static std::map jl_value_to_llvm; -JL_DLLEXPORT std::map jl_llvm_to_jl_value; - -// In imaging mode, cache a fast mapping of Function * to code address -// because this is queried in the hot path -static std::map emitted_function_symtab; - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static Function *function_proto(Function *F) -{ - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName()); - NewF->setAttributes(AttributeSet()); - - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); - +// GlobalVariable *gv = stringConstants[txt]; +// // in inference, we can not share string constants between +// // modules as there might be multiple compiles on the stack +// // with calls in between them. +// if (gv == NULL) { +// std::stringstream ssno; +// std::string vname; +// ssno << strno; +// vname += "_j_str"; +// vname += ssno.str(); +// gv = new GlobalVariable(*jl_builderModule, +// T_int8, +// true, +// GlobalVariable::ExternalLinkage, +// NULL, +// vname); +// stringConstants[txt] = global_proto(gv); +// jl_sym_t *sym = jl_symbol(txt.c_str()); +// *(jl_sym_t**)jl_emit_and_add_to_shadow(gv, sym) = sym; +// strno++; +// } +// else { +// gv = prepare_global(gv); +// } +// return builder.CreateConstInBoundsGEP1_32(T_int8, +// builder.CreateLoad(T_pint8, ConstantExpr::getBitCast(gv, T_ppint8)), +// sizeof(jl_sym_t)); +} + +static Value *GetStringPtr(llvm::IRBuilder <> *builder, GlobalValue *GV, StringRef Name) +{ + Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); + Value *Args[] = { zero, zero }; #ifdef LLVM37 - // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway, so clear them again manually - NewF->setPersonalityFn(nullptr); -#endif - - return NewF; -} - -class FunctionMover : public ValueMaterializer -{ -public: - FunctionMover(llvm::Module *dest,llvm::Module *src) : - ValueMaterializer(), VMap(), destModule(dest), srcModule(src), - LazyFunctions(0) - { - } - ValueToValueMapTy VMap; - llvm::Module *destModule; - llvm::Module *srcModule; - std::vector LazyFunctions; - - Function *CloneFunctionProto(Function *F) - { - assert(!F->isDeclaration()); - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - destModule); - LazyFunctions.push_back(F); - VMap[F] = NewF; - return NewF; - } - - void CloneFunctionBody(Function *F) - { - Function *NewF = (Function*)(Value*)VMap[F]; - assert(NewF != NULL); - - Function::arg_iterator DestI = NewF->arg_begin(); - for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { - DestI->setName(I->getName()); // Copy the name over... - VMap[&*I] = &*(DestI++); // Add mapping to VMap - } - - #ifdef LLVM36 - // Clone debug info - Not yet public API - // llvm::CloneDebugInfoMetadata(NewF,F,VMap); - #endif - - SmallVector Returns; - llvm::CloneFunctionInto(NewF,F,VMap,true,Returns,"",NULL,NULL,this); - } - - Function *CloneFunction(Function *F) - { - Function *NewF = (llvm::Function*)MapValue(F,VMap,RF_None,NULL,this); - ResolveLazyFunctions(); - return NewF; - } - - void ResolveLazyFunctions() - { - while (!LazyFunctions.empty()) { - Function *F = LazyFunctions.back(); - LazyFunctions.pop_back(); - - CloneFunctionBody(F); - } - } - - Value *InjectFunctionProto(Function *F) - { - Function *NewF = destModule->getFunction(F->getName()); - if (!NewF) { - NewF = function_proto(F); - destModule->getFunctionList().push_back(NewF); - } - return NewF; - } - -#ifdef LLVM38 - virtual Value *materializeDeclFor(Value *V) + return builder->CreateInBoundsGEP(GV->getValueType(), GV, Args, Name); #else - virtual Value *materializeValueFor (Value *V) + return builder->CreateInBoundsGEP(GV, Args, Name); #endif - { - Function *F = dyn_cast(V); - if (F) { - if (F->isIntrinsic()) { - return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); - } - if (F->isDeclaration() || F->getParent() != destModule) { - if (F->getName().empty()) - return CloneFunctionProto(F); - Function *shadow = srcModule->getFunction(F->getName()); - if (shadow != NULL && !shadow->isDeclaration()) { - // Not truly external - // Check whether we already emitted it once - if (emitted_function_symtab.find(shadow) != emitted_function_symtab.end()) - return InjectFunctionProto(F); - - Function *oldF = destModule->getFunction(F->getName()); - if (oldF) - return oldF; - -#ifndef USE_ORCJIT - // Also check if this function is pending in any other module - if (jl_ExecutionEngine->FindFunctionNamed(F->getName().data())) - return InjectFunctionProto(F); -#endif - - return CloneFunctionProto(shadow); - } - else if (!F->isDeclaration()) { - return CloneFunctionProto(F); - } - } - // Still a declaration and still in a different module - if (F->isDeclaration() && F->getParent() != destModule) { - // Create forward declaration in current module - return InjectFunctionProto(F); - } - } - else if (isa(V)) { - GlobalVariable *GV = cast(V); - assert(GV != NULL); - GlobalVariable *oldGV = destModule->getGlobalVariable(GV->getName()); - if (oldGV != NULL) - return oldGV; - GlobalVariable *newGV = new GlobalVariable(*destModule, - GV->getType()->getElementType(), - GV->isConstant(), - GlobalVariable::ExternalLinkage, - NULL, - GV->getName()); - newGV->copyAttributesFrom(GV); - if (GV->isDeclaration()) - return newGV; - if (!GV->getName().empty()) { - uint64_t addr = jl_ExecutionEngine->getGlobalValueAddress(GV->getName()); - if (addr != 0) { - newGV->setExternallyInitialized(true); - return newGV; - } - } - std::map::iterator it; - it = jl_llvm_to_jl_value.find(GV); - if (it != jl_llvm_to_jl_value.end()) { - newGV->setInitializer(Constant::getIntegerValue(GV->getType()->getElementType(),APInt(sizeof(void*)*8,(intptr_t)it->second))); - newGV->setConstant(true); - } - else if (GV->hasInitializer()) { - Value *C = MapValue(GV->getInitializer(),VMap,RF_None,NULL,this); - newGV->setInitializer(cast(C)); - } - return newGV; - } - return NULL; - }; -}; -#else -static Function *function_proto(Function *F) -{ - return F; +// return GV; } -#endif + +// --- Debug info --- #ifdef LLVM37 static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed = false) @@ -569,218 +183,6 @@ static Value *literal_static_pointer_val(const void *p, Type *t) #endif } -static std::vector jl_sysimg_gvars; -static std::vector jl_sysimg_fvars; - -extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) -{ - // map a jl_value_t memory location to a GlobalVariable - std::map::iterator it; - it = jl_value_to_llvm.find(p); - if (it == jl_value_to_llvm.end()) - return 0; - return it->second.index; -} - -#ifdef HAVE_CPUID -extern "C" { - extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); -} -#endif - -static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, - const char *sysimg_data, size_t sysimg_len) -{ - ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); - addComdat(new GlobalVariable(*mod, - gvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), - "jl_sysimg_gvars")); - ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); - addComdat(new GlobalVariable(*mod, - fvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), - "jl_sysimg_fvars")); - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size,globalUnique+1), - "jl_globalUnique")); -#ifdef JULIA_ENABLE_THREADING - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, jltls_states_func_idx), - "jl_ptls_states_getter_idx")); -#endif - - Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); - addComdat(new GlobalVariable(*mod, - feature_string->getType(), - true, - GlobalVariable::ExternalLinkage, - feature_string, - "jl_sysimg_cpu_target")); - -#ifdef HAVE_CPUID - // For native also store the cpuid - if (strcmp(jl_options.cpu_target,"native") == 0) { - uint32_t info[4]; - - jl_cpuid((int32_t*)info, 1); - addComdat(new GlobalVariable(*mod, - T_int64, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), - "jl_sysimg_cpu_cpuid")); - } -#endif - - if (sysimg_data) { - Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); - addComdat(new GlobalVariable(*mod, data->getType(), true, - GlobalVariable::ExternalLinkage, - data, "jl_system_image_data")); - Constant *len = ConstantInt::get(T_size, sysimg_len); - addComdat(new GlobalVariable(*mod, len->getType(), true, - GlobalVariable::ExternalLinkage, - len, "jl_system_image_size")); - } -} - -static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, - bool dump_as_bc) -{ -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif -#ifdef JL_DEBUG_BUILD - verifyModule(*shadow_module); -#endif - -#ifdef LLVM36 - std::error_code err; - StringRef fname_ref = StringRef(fname); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); -#elif defined(LLVM35) - std::string err; - raw_fd_ostream OS(fname, err, sys::fs::F_None); -#else - std::string err; - raw_fd_ostream OS(fname, err); -#endif -#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone - raw_fd_ostream& FOS(OS); -#else - formatted_raw_ostream FOS(OS); -#endif - - // We don't want to use MCJIT's target machine because - // it uses the large code model and we may potentially - // want less optimizations there. - Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); -#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::COFF); -#else - TheTriple.setEnvironment(Triple::UnknownEnvironment); -#endif -#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::MachO); -#else - TheTriple.setEnvironment(Triple::MachO); -#endif -#endif -#ifdef LLVM35 - std::unique_ptr -#else - OwningPtr -#endif - TM(jl_TargetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - jl_TargetMachine->getTargetCPU(), - jl_TargetMachine->getTargetFeatureString(), - jl_TargetMachine->Options, -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - Reloc::PIC_, -#else - jit_model ? Reloc::PIC_ : Reloc::Default, -#endif - jit_model ? CodeModel::JITDefault : CodeModel::Default, - CodeGenOpt::Aggressive // -O3 - )); - -#ifdef LLVM38 - legacy::PassManager PM; -#else - PassManager PM; -#endif -#ifndef LLVM37 - PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); -#else - PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); -#endif -#ifdef LLVM37 -// No DataLayout pass needed anymore. -#elif defined(LLVM36) - PM.add(new DataLayoutPass()); -#elif defined(LLVM35) - PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); -#else - PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); -#endif - - addOptimizationPasses(&PM); - if (!dump_as_bc) { - if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { - jl_error("Could not generate obj file for this target"); - } - } - - // now copy the module, since PM.run may modify it - ValueToValueMapTy VMap; -#ifdef LLVM38 - Module *clone = CloneModule(shadow_module, VMap).release(); -#else - Module *clone = CloneModule(shadow_module, VMap); -#endif -#ifdef LLVM37 - // Reset the target triple to make sure it matches the new target machine - clone->setTargetTriple(TM->getTargetTriple().str()); -#ifdef LLVM38 - clone->setDataLayout(TM->createDataLayout()); -#else - clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); -#endif -#endif - - // add metadata information - jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); - - // do the actual work - finalize_gc_frame(clone); - PM.run(*clone); - if (dump_as_bc) - WriteBitcodeToFile(clone, FOS); - delete clone; -} - -static int32_t jl_assign_functionID(Function *functionObject, int specsig) -{ - // give the function an index in the constant lookup table - if (!imaging_mode) - return 0; - jl_sysimg_fvars.push_back(ConstantExpr::getBitCast(functionObject, T_pvoidfunc)); - return jl_sysimg_fvars.size(); -} static Value *julia_gv(const char *cname, void *addr) { @@ -794,24 +196,11 @@ static Value *julia_gv(const char *cname, void *addr) std::stringstream gvname; gvname << cname << globalUnique++; // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - ConstantPointerNull::get((PointerType*)T_pjlvalue), gvname.str()); + GlobalVariable *gv = new GlobalVariable(*jl_builderModule, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); addComdat(gv); - - // make the pointer valid for this session -#ifdef USE_MCJIT - jl_llvm_to_jl_value[gv] = addr; -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(gv); - *p = addr; -#endif - // make the pointer valid for future sessions - jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(gv, T_psize)); - jl_value_llvm gv_struct; - gv_struct.gv = prepare_global(gv); - gv_struct.index = jl_sysimg_gvars.size(); - jl_value_to_llvm[addr] = gv_struct; + *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; return builder.CreateLoad(gv); } @@ -1183,11 +572,8 @@ static Value *emit_datatype_isbitstype(Value *dt) static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; builder.CreateCall(prepare_call(jlerror_func), - builder.CreateGEP(stringConst(txt), - ArrayRef(zeros))); + GetStringPtr(&builder, stringConst(txt), "just_emit_error")); } static void emit_error(const std::string &txt, jl_codectx_t *ctx) @@ -1252,12 +638,8 @@ static void null_pointer_check(Value *v, jl_codectx_t *ctx) static void emit_type_error(const jl_cgval_t &x, jl_value_t *type, const std::string &msg, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; - Value *fname_val = builder.CreateGEP(stringConst(ctx->funcName), - ArrayRef(zeros)); - Value *msg_val = builder.CreateGEP(stringConst(msg), - ArrayRef(zeros)); + Value *fname_val = GetStringPtr(&builder, stringConst(ctx->funcName), "fname_val"); + Value *msg_val = GetStringPtr(&builder, stringConst(msg), "msg_val"); #ifdef LLVM37 builder.CreateCall(prepare_call(jltypeerror_func), { fname_val, msg_val, diff --git a/src/codegen.cpp b/src/codegen.cpp index e06211ee5da30..7175cf80449b5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -12,6 +12,9 @@ // also MCJIT debugging support on windows needs ELF (currently) #define FORCE_ELF #endif +#if defined(_CPU_X86_) +#define JL_NEED_FLOATTEMP_VAR 1 +#endif #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS @@ -198,31 +201,17 @@ namespace llvm { // for image reloading static bool imaging_mode = false; -#include "jitlayers.cpp" - -#ifdef USE_ORCJIT -JuliaOJIT *jl_ExecutionEngine; -#else -ExecutionEngine *jl_ExecutionEngine; -#endif - -#ifdef USE_MCJIT -static Module *shadow_module; -static Module *builtins_module; -static Module *active_module; -#define jl_Module (builder.GetInsertBlock()->getParent()->getParent()) -#else -static Module *jl_Module; -#define shadow_module jl_Module -#define active_module jl_Module -#define builtins_module jl_Module -#endif +static Module *shadow_output; +#define jl_Module ctx->f->getParent() +#define jl_builderModule builder.GetInsertBlock()->getParent()->getParent() static MDBuilder *mbuilder; static std::map argNumberStrings; +#ifndef USE_ORCJIT #ifdef LLVM38 -static legacy::FunctionPassManager *FPM; +static legacy::PassManager *PM; #else -static FunctionPassManager *FPM; +static PassManager *PM; +#endif #endif #ifdef LLVM37 @@ -322,12 +311,6 @@ static GlobalVariable *jltrue_var; static GlobalVariable *jlfalse_var; static GlobalVariable *jlemptysvec_var; static GlobalVariable *jlemptytuple_var; -#if defined(_CPU_X86_) -#define JL_NEED_FLOATTEMP_VAR 1 -#endif -#ifdef JL_NEED_FLOATTEMP_VAR -static GlobalVariable *jlfloattemp_var; -#endif static GlobalVariable *jldiverr_var; static GlobalVariable *jlundeferr_var; static GlobalVariable *jldomerr_var; @@ -438,6 +421,11 @@ static std::vector three_pvalue_llvmt; static std::map builtin_func_map; // --- code generation --- +extern "C" { + int globalUnique = 0; +} + +#include "jitlayers.cpp" // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { @@ -556,13 +544,6 @@ typedef struct { jl_value_t *ty; } jl_arrayvar_t; -// Keeps tracks of all functions and compile units created during this cycle -// to be able to atomically add them to a module. -typedef struct { - std::vector functions; - std::vector CUs; -} jl_cyclectx_t; - // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. typedef struct { @@ -595,8 +576,6 @@ typedef struct { llvm::DIBuilder *dbuilder; bool debug_enabled; std::vector to_inline; - - jl_cyclectx_t *cyclectx; } jl_codectx_t; typedef struct { @@ -624,8 +603,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol=false); static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); -static void finalize_gc_frame(Function *F); -static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr m); static GlobalVariable *prepare_global(GlobalVariable *G); // --- convenience functions for tagging llvm values with julia types --- @@ -717,10 +695,6 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) // --- utilities --- -extern "C" { - int globalUnique = 0; -} - static void emit_write_barrier(jl_codectx_t*, Value*, Value*); #include "cgutils.cpp" @@ -822,17 +796,16 @@ void jl_dump_compiles(void *s) // --- entry point --- //static int n_emit=0; -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx); -static void jl_finalize_module(Module *m); +static Module *emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations); void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataLayout &DL); //static int n_compile=0; -static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) +static Module *to_function(jl_lambda_info_t *li) { JL_LOCK(codegen); JL_SIGATOMIC_BEGIN(); assert(!li->inInference); + li->inCompile = 1; BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; DebugLoc olddl = builder.getCurrentDebugLocation(); bool last_n_c = nested_compile; @@ -840,30 +813,12 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) last_time = jl_hrtime(); nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); + Module *m = NULL; Function *f = NULL, *specf = NULL; JL_TRY { - jl_llvm_functions_t definitions; - #if defined(USE_MCJIT) || defined(USE_ORCJIT) - jl_cyclectx_t *newcyclectx = cyclectx; - if (!newcyclectx) - newcyclectx = new jl_cyclectx_t; - emit_function(li, &li->functionObjects, &definitions, newcyclectx); - // If we're the root of the cycle, realize all functions - if (!cyclectx) { - realize_cycle(newcyclectx); - delete newcyclectx; - } - #else - emit_function(li, &li->functionObjects, &definitions, NULL); - #endif - f = (llvm::Function*)definitions.functionObject; - specf = (llvm::Function*)definitions.specFunctionObject; - li->functionID = jl_assign_functionID(f, 0); - if (specf) - li->specFunctionID = jl_assign_functionID(specf, 1); - if (f->getFunctionType() != jl_func_sig) - // mark the pointer as jl_fptr_sparam_t calling convention - li->jlcall_api = 1; + m = emit_function(li, &li->functionObjects); + f = (Function*)li->functionObjects.functionObject; + specf = (Function*)li->functionObjects.specFunctionObject; //n_emit++; } JL_CATCH { @@ -875,45 +830,40 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; JL_SIGATOMIC_END(); JL_UNLOCK(codegen); jl_rethrow_with_add("error compiling %s", jl_symbol_name(li->name)); } - assert(f != NULL); - const DataLayout &DL = -#ifdef LLVM35 - f->getParent()->getDataLayout(); -#else - *jl_data_layout; -#endif + assert(m != NULL); + jl_finalize_module(std::unique_ptr(m)); + // record that this function name came from this linfo, // so we can build a reverse mapping for debug-info. if (li->name != anonymous_sym) { + const DataLayout &DL = +#ifdef LLVM35 + m->getDataLayout(); +#else + *jl_data_layout; +#endif // but don't remember anonymous symbols because // they may not be rooted in the gc for the life of the program, // and the runtime doesn't notify us when the code becomes unreachable :( jl_add_linfo_in_flight((specf ? specf : f)->getName(), li, DL); } -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(f); - if (specf) finalize_gc_frame(specf); -#ifdef JL_DEBUG_BUILD - if (verifyFunction(*f, PrintMessageAction) || - (specf && verifyFunction(*specf, PrintMessageAction))) { - f->dump(); - if (specf) specf->dump(); - gc_debug_critical_error(); - abort(); - } -#endif - FPM->run(*f); - if (specf) FPM->run(*specf); -#endif + li->functionID = jl_assign_functionID(f, 0); + if (specf) + li->specFunctionID = jl_assign_functionID(specf, 1); + if (f->getFunctionType() != jl_func_sig) + // mark the pointer as jl_fptr_sparam_t calling convention + li->jlcall_api = 1; if (old != NULL) { builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; nested_compile = last_n_c; jl_gc_inhibit_finalizers(nested_compile); JL_UNLOCK(codegen); @@ -925,7 +875,7 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) jl_printf(dump_compiles_stream, "\"\n"); last_time = this_time; } - return specf ? specf : f; + return m; } #ifndef LLVM37 @@ -976,185 +926,69 @@ static void jl_setup_module(Module *m) #endif } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static void jl_finalize_module(Module *m) +static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr uniquem) { -#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(USE_MCJIT) - ArrayType *atype = ArrayType::get(T_uint32,3); // want 4-byte alignment of 12-bytes of data - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); -#endif + Module *m = uniquem.release(); finalize_gc_frame(m); - assert(jl_ExecutionEngine); -#if defined(LLVM36) && !defined(USE_ORCJIT) - jl_ExecutionEngine->addModule(std::unique_ptr(m)); -#else - jl_ExecutionEngine->addModule(m); +#if !defined(USE_ORCJIT) +#ifdef LLVM33 +#ifdef JL_DEBUG_BUILD + if (verifyModule(*m, PrintMessageAction)) { + m->dump(); + gc_debug_critical_error(); + abort(); + } #endif -} - -#if !defined(USE_ORCJIT) && defined(JL_DEBUG_BUILD) -static void writeRecoveryFile(llvm::Module *mod) -{ - std::error_code err; - mod->dump(); - std::cout << "Julia emitted a broken LLVM module (about to be __jl_dump.bc)." - << "Please file a bug report.\n" - << "If the module writing below fails," - << "please include the textual representation printed above this error."; - StringRef fname_ref = StringRef("__jl_dump.bc"); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); - WriteBitcodeToFile(mod,OS); - OS.flush(); - gc_debug_critical_error(); - abort(); -} #endif + PM->run(*m); +#endif +#ifdef USE_MCJIT + // record the function names that are part of this Module + // so it can be added to the JIT when needed + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) + module_for_fname[F->getName()] = m; + } +#endif + jl_add_to_shadow(m); +} -static uint64_t getAddressForOrCompileFunction(llvm::Function *llvmf) +static uint64_t getAddressForFunction(llvm::Function *llvmf) { - #ifdef JL_DEBUG_BUILD +#ifdef JL_DEBUG_BUILD llvm::raw_fd_ostream out(1,false); - #endif - Function *ActiveF = active_module->getFunction(llvmf->getName()); - // Must have been in a prior module. Safe to ask the execution engine - // to emit it. - if (!ActiveF || ActiveF->isDeclaration()) - return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - if (!imaging_mode) { - realize_pending_globals(); - #ifndef USE_ORCJIT - #ifdef JL_DEBUG_BUILD -#ifdef LLVM38 - Module *backup = llvm::CloneModule(active_module).release(); +#endif +#ifdef USE_MCJIT + jl_finalize_function(llvmf, NULL); + return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); #else - Module *backup = llvm::CloneModule(active_module); + return (uint64_t)jl_ExecutionEngine->getPointerToFunction( + cast(shadow_output->getNamedValue(llvmf->getName()))); #endif - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - #endif - for (auto &F : active_module->functions()) { - if (F.isDeclaration()) - continue; - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - finalize_gc_frame(&F); - FPM->run(F); - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - } - #ifdef JL_DEBUG_BUILD - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - delete backup; - #endif - #endif - jl_finalize_module(active_module); - } - uint64_t addr = jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - assert(addr != 0); - if (!imaging_mode) { - active_module = new Module("julia", jl_LLVMContext); - jl_setup_module(active_module); - } - return addr; } -#endif extern "C" void jl_generate_fptr(jl_lambda_info_t *li) { JL_LOCK(codegen); // objective: assign li->fptr assert(li->functionObjects.functionObject); + assert(!li->inCompile); if (li->fptr == NULL) { JL_SIGATOMIC_BEGIN(); - #ifdef USE_MCJIT - if (imaging_mode) { - // see if it has been emitted already (as part of compiling something else) - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - if (li->fptr == NULL) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - mover.CloneFunction((Function*)li->functionObjects.functionObject); - if (li->functionObjects.specFunctionObject != NULL) - mover.CloneFunction((Function*)li->functionObjects.specFunctionObject); - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { - list->data()[i].f = mover.CloneFunction(list->data()[i].f); - } - } - jl_finalize_module(m); - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - } - } - else { - li->fptr = (jl_fptr_t)getAddressForOrCompileFunction((Function*)li->functionObjects.functionObject); - } - #else - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.functionObject); - #endif - + li->fptr = (jl_fptr_t)getAddressForFunction((Function*)li->functionObjects.functionObject); assert(li->fptr != NULL); -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.functionObject)->deleteBody(); -#endif - - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { -#ifdef USE_MCJIT - (void)getAddressForOrCompileFunction(list->data()[i].f); -#else - (void)jl_ExecutionEngine->getPointerToFunction(list->data()[i].f); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) { - list->data()[i].f->deleteBody(); - } -#endif - } - } - - if (li->functionObjects.specFunctionObject != NULL) { -#ifdef USE_MCJIT - if (imaging_mode) - (void)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.specFunctionObject)->getName()); - else - (void)getAddressForOrCompileFunction((Function*)li->functionObjects.specFunctionObject); -#else - (void)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.specFunctionObject); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.specFunctionObject)->deleteBody(); -#endif - } JL_SIGATOMIC_END(); } JL_UNLOCK(codegen); } -extern "C" void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx) +extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { if (li->functionObjects.functionObject == NULL) { // objective: assign li->functionObject - li->inCompile = 1; - (void)to_function(li, (jl_cyclectx_t *)cyclectx); - li->inCompile = 0; + (void)to_function(li); } } @@ -1212,7 +1046,7 @@ static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tuplet } } - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt, NULL); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt); if (li != NULL) { for(i=1; i < nargs+1; i++) { jl_value_t *speci = jl_nth_slot_type(li->specTypes, i); @@ -1256,26 +1090,8 @@ void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) } assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - assert(llvmf); JL_GC_POP(); - -#ifdef USE_MCJIT - if (uint64_t addr = getAddressForOrCompileFunction(llvmf)) - return (void*)(intptr_t)addr; - if (llvmf->getParent() == shadow_module) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - (void)mover.CloneFunction(llvmf); - jl_finalize_module(m); - } -#endif -#ifdef USE_MCJIT - return (void*)getAddressForOrCompileFunction(llvmf); -#else - return jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif + return (void*)getAddressForFunction(llvmf); } @@ -1296,31 +1112,23 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); if (llvmf) { - Function *active_llvmf = active_module->getFunction(llvmf->getName()); - // In imaging mode, and in most cases in JIT mode (where the wrapper is specifically - // compiled for this function), we can simply use a global alias. - if (active_llvmf) { - #ifndef LLVM35 - new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, llvmf->getParent()); - #elif defined(LLVM37) && !defined(LLVM38) - GlobalAlias::create(cast(llvmf->getType()), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #else + // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) + uint64_t Addr = getAddressForFunction(llvmf); + // emit the function pointer and set up an alias in the execution engine +#if defined(USE_ORCJIT) || defined(USE_MCJIT) + jl_ExecutionEngine->addGlobalMapping(name, Addr); + if (imaging_mode) { + llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); +#endif + // in imaging_mode, also need to add the alias to the shadow_module +#if defined(LLVM38) GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #endif - } else { - // Otherwise we use a global mapping - assert(!imaging_mode); -#if defined(USE_ORCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - (void*)getAddressForOrCompileFunction(llvmf)); -#elif defined(USE_MCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - getAddressForOrCompileFunction(llvmf)); + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); +#elif defined(LLVM37) + GlobalAlias::create(cast(llvmf->getType()), + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #else - jl_ExecutionEngine->addGlobalMapping(llvmf, - jl_ExecutionEngine->getPointerToFunction(llvmf)); + new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #endif } } @@ -1334,7 +1142,7 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g jl_lambda_info_t *linfo = NULL; JL_GC_PUSH2(&linfo, &tt); if (tt != NULL) { - linfo = jl_get_specialization1(tt, NULL); + linfo = jl_get_specialization1(tt); if (linfo == NULL) { linfo = jl_method_lookup_by_type(jl_gf_mtable(f), tt, 0, 0); if (linfo == NULL) { @@ -1353,62 +1161,40 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g } if (!getdeclarations) { - Function *llvmDecl = NULL; - if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) - llvmDecl = (Function*)linfo->functionObjects.specFunctionObject; - else - llvmDecl = (Function*)linfo->functionObjects.functionObject; -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - Function *llvmf = llvmDecl ? active_module->getFunction(llvmDecl->getName()) : NULL; - // Note that in either case, we need to run the FPM manually, - // since this is now usually done as part of object emission -#else - Function *llvmf = llvmDecl && !llvmDecl->isDeclaration() ? llvmDecl : NULL; -#endif - if (!llvmf) { - Function *other; - jl_llvm_functions_t declarations; - emit_function(linfo, NULL, &declarations, NULL); - if (getwrapper || !declarations.specFunctionObject) { - llvmf = (llvm::Function*)declarations.functionObject; - other = (llvm::Function*)declarations.specFunctionObject; - } - else { - llvmf = (llvm::Function*)declarations.specFunctionObject; - other = (llvm::Function*)declarations.functionObject; - } - if (other) - other->eraseFromParent(); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); -#endif - llvmf->removeFromParent(); - if (llvmDecl) - llvmf->setName(llvmDecl->getName()); + // emit this function into a new module + Function *f, *specf; + jl_llvm_functions_t declarations; + Module *m = emit_function(linfo, &declarations); + f = (llvm::Function*)declarations.functionObject; + specf = (llvm::Function*)declarations.specFunctionObject; + // swap declarations for definitions and destroy declarations + if (specf) { + Function *temp = cast(m->getNamedValue(specf->getName())); + delete specf; + specf = temp; + } + if (specf) { + Function *temp = cast(m->getNamedValue(f->getName())); + delete f; + f = temp; + } + Function *specf_decl = (Function*)linfo->functionObjects.specFunctionObject; + if (specf_decl) { + specf->setName(specf_decl->getName()); + } + Function *f_decl = (Function*)linfo->functionObjects.functionObject; + if (f_decl) { + f->setName(f_decl->getName()); + } + JL_GC_POP(); + if (getwrapper || !specf) { + return f; } else { - ValueToValueMapTy VMap; - Function *oldllvmf = llvmf; - llvmf = CloneFunction(llvmf, VMap, false); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - active_module->getFunctionList().push_back(llvmf); - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); - llvmf->removeFromParent(); - llvmf->setName(oldllvmf->getName()); -#else - (void)oldllvmf; -#endif + return specf; } - JL_GC_POP(); - return llvmf; - } - if (linfo->functionObjects.functionObject == NULL) { - jl_compile_linfo(linfo, NULL); } + jl_compile_linfo(linfo); Function *llvmf; if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjects.specFunctionObject; @@ -1420,16 +1206,6 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g return llvmf; } -extern "C" JL_DLLEXPORT -uint64_t jl_get_llvm_fptr(llvm::Function *llvmf) -{ -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - return getAddressForOrCompileFunction(llvmf); -#else - return (uint64_t)jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif -} - extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_module) { @@ -1437,57 +1213,50 @@ const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump llvm::raw_string_ostream stream(code); Function *llvmf = dyn_cast((Function*)f); - if (!llvmf) - jl_error("jl_dump_function_ir: Expected Function*"); + if (!llvmf || (!llvmf->isDeclaration() && !llvmf->getParent())) + jl_error("jl_dump_function_ir: Expected Function* in a temporary Module"); - if (llvmf->isDeclaration()) { - // print the function declaration plain + if (!llvmf->getParent()) { + // print the function declaration as-is llvmf->print(stream); } else { - if (llvmf->getParent()) - jl_error("jl_dump_function_ir requires a parentless clone"); - // Put the function in a module - Module *m = new Module(llvmf->getName(), jl_LLVMContext); - jl_setup_module(m); - m->getFunctionList().push_back(llvmf); - Function *f2 = llvmf; + Module *m = llvmf->getParent(); if (strip_ir_metadata) { - // strip metadata from the copy - Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); - // iterate over all basic blocks in the function - for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { - BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); - // iterate over instructions in basic block - for (; f2_il != (*f2_bb).getInstList().end(); ) { - Instruction *inst = &*f2_il++; - // remove dbg.declare and dbg.value calls - if (isa(inst) || isa(inst)) { - inst->eraseFromParent(); - continue; - } + // strip metadata from all instructions in the module + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *f2 = &*I; + Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); + // iterate over all basic blocks in the function + for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { + BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); + // iterate over instructions in basic block + for (; f2_il != (*f2_bb).getInstList().end(); ) { + Instruction *inst = &*f2_il++; + // remove dbg.declare and dbg.value calls + if (isa(inst) || isa(inst)) { + inst->eraseFromParent(); + continue; + } - SmallVector, 4> MDForInst; - inst->getAllMetadata(MDForInst); - SmallVector, 4>::iterator md_iter = MDForInst.begin(); + SmallVector, 4> MDForInst; + inst->getAllMetadata(MDForInst); + SmallVector, 4>::iterator md_iter = MDForInst.begin(); - // iterate over all metadata kinds and set to NULL to remove - for (; md_iter != MDForInst.end(); ++md_iter) { - inst->setMetadata((*md_iter).first, NULL); + // iterate over all metadata kinds and set to NULL to remove + for (; md_iter != MDForInst.end(); ++md_iter) { + inst->setMetadata((*md_iter).first, NULL); + } } } } } if (dump_module) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif m->print(stream, NULL); } - else - f2->print(stream); - f2->eraseFromParent(); - m->dropAllReferences(); + else { + llvmf->print(stream); + } delete m; } @@ -1581,18 +1350,15 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) // Dump assembly code uint64_t symsize = 0; int64_t slide = 0, section_slide = 0; + uint64_t fptr = getAddressForFunction(llvmf); #ifdef USE_MCJIT - uint64_t fptr = getAddressForOrCompileFunction(llvmf); -#ifdef USE_ORCJIT // Look in the system image as well if (fptr == 0) - fptr = jl_ExecutionEngine->findSymbol( - jl_ExecutionEngine->mangle(llvmf->getName()), true).getAddress(); -#endif + fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(llvmf)); llvm::DIContext *context = NULL; llvm::DIContext *&objcontext = context; #else - uint64_t fptr = (uintptr_t)jl_ExecutionEngine->getPointerToFunction(llvmf); std::vector context; llvm::DIContext *objcontext = NULL; #endif @@ -1650,24 +1416,24 @@ static logdata_t coverageData; static void coverageVisitLine(StringRef filename, int line) { - if (filename == "" || filename == "none" || filename == "no file") - return; - logdata_t::iterator it = coverageData.find(filename); - if (it == coverageData.end()) { - coverageData[filename] = std::vector(0); - } - std::vector &vec = coverageData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "lcnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - ConstantInt::get(T_int64,1)), - v, true); +// if (filename == "" || filename == "none" || filename == "no file") +// return; +// logdata_t::iterator it = coverageData.find(filename); +// if (it == coverageData.end()) { +// coverageData[filename] = std::vector(0); +// } +// std::vector &vec = coverageData[filename]; +// if (vec.size() <= (size_t)line) +// vec.resize(line+1, NULL); +// if (vec[line] == NULL) { +// vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, +// GlobalVariable::InternalLinkage, +// ConstantInt::get(T_int64,0), "lcnt")); +// } +// GlobalVariable *v = prepare_global(vec[line]); +// builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), +// ConstantInt::get(T_int64,1)), +// v, true); } extern "C" int isabspath(const char *in); @@ -1700,11 +1466,7 @@ static void write_log_data(logdata_t &logData, const char *extension) if ((size_t)l < values.size()) { GlobalVariable *gv = values[l]; if (gv) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress(gv->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(gv); -#endif + int *p = (int*)jl_get_global(gv); value = *p; } } @@ -1739,30 +1501,30 @@ static logdata_t mallocData; static void mallocVisitLine(StringRef filename, int line) { - if (filename == "" || filename == "none" || filename == "no file") { - jl_gc_sync_total_bytes(); - return; - } - logdata_t::iterator it = mallocData.find(filename); - if (it == mallocData.end()) { - mallocData[filename] = std::vector(0); - } - std::vector &vec = mallocData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "bytecnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - builder.CreateCall(prepare_call(diff_gc_total_bytes_func) -#ifdef LLVM37 - , {} -#endif - )), - v, true); +// if (filename == "" || filename == "none" || filename == "no file") { +// jl_gc_sync_total_bytes(); +// return; +// } +// logdata_t::iterator it = mallocData.find(filename); +// if (it == mallocData.end()) { +// mallocData[filename] = std::vector(0); +// } +// std::vector &vec = mallocData[filename]; +// if (vec.size() <= (size_t)line) +// vec.resize(line+1, NULL); +// if (vec[line] == NULL) { +// vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, +// GlobalVariable::InternalLinkage, +// ConstantInt::get(T_int64,0), "bytecnt")); +// } +// GlobalVariable *v = prepare_global(vec[line]); +// builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), +// builder.CreateCall(prepare_call(diff_gc_total_bytes_func) +//#ifdef LLVM37 +// , {} +//#endif +// )), +// v, true); } // Resets the malloc counts. Needed to avoid including memory usage @@ -1775,11 +1537,7 @@ extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void) std::vector::iterator itb; for (itb = bytes.begin(); itb != bytes.end(); itb++) { if (*itb) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress((*itb)->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(*itb); -#endif + int *p = (int*)jl_get_global(*itb); *p = 0; } } @@ -2943,11 +2701,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval jl_value_t *jlretty = li->rettype; bool retboxed; (void)julia_type_to_llvm(jlretty, &retboxed); - Function *cf = (Function*)li->functionObjects.specFunctionObject; - if (!cf->getParent() || (cf->getParent() == builtins_module && - builtins_module != active_module)) { // Call cycle - prepare_call(cf); - } + Function *cf = cast(prepare_call((Function*)li->functionObjects.specFunctionObject)); FunctionType *cft = cf->getFunctionType(); size_t nfargs = cft->getNumParams(); Value **argvals = (Value**) alloca(nfargs*sizeof(Value*)); @@ -3056,7 +2810,7 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_sprint(args[0]), jl_sprint((jl_value_t*)aty)); }*/ - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty, ctx->cyclectx); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty); if (li != NULL) { assert(li->functionObjects.functionObject != NULL); theFptr = (Value*)li->functionObjects.functionObject; @@ -3140,11 +2894,12 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, b = jl_get_binding(m, s); if (b == NULL) { // var not found. switch to delayed lookup. + std::stringstream name; + name << "delayedvar" << globalUnique++; Constant *initnul = ConstantPointerNull::get((PointerType*)T_pjlvalue); - GlobalVariable *bindinggv = - prepare_global(new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, GlobalVariable::PrivateLinkage, - initnul, "delayedvar")); + GlobalVariable *bindinggv = new GlobalVariable(*ctx->f->getParent(), T_pjlvalue, + false, GlobalVariable::PrivateLinkage, + initnul, name.str()); Value *cachedval = builder.CreateLoad(bindinggv); BasicBlock *have_val = BasicBlock::Create(jl_LLVMContext, "found"), *not_found = BasicBlock::Create(jl_LLVMContext, "notfound"); @@ -3817,11 +3572,11 @@ static void finalize_gc_frame(Function *F) static void finalize_gc_frame(Module *m) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto &F : m->functions()) { - if (F.isDeclaration()) + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (F->isDeclaration()) continue; - finalize_gc_frame(&F); + finalize_gc_frame(F); } #ifndef JULIA_ENABLE_THREADING m->getFunction("jl_get_ptls_states")->eraseFromParent(); @@ -3830,7 +3585,6 @@ static void finalize_gc_frame(Module *m) m->getFunction("julia.gc_root_kill")->eraseFromParent(); m->getFunction("julia.gc_store")->eraseFromParent(); m->getFunction("julia.jlcall_frame_decl")->eraseFromParent(); -#endif } // here argt does not include the leading function type argument @@ -3875,7 +3629,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v if (fargt.size() + sret != fargt_sig.size()) jl_error("va_arg syntax not allowed for cfunction argument list"); - jl_compile_linfo(lam, NULL); + jl_compile_linfo(lam); if (!lam->functionObjects.functionObject) { jl_errorf("error compiling %s while creating cfunction", jl_symbol_name(lam->name)); @@ -3892,9 +3646,11 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); // no allocations expected between the top of this function (when last scanned lam->cFunctionList) and here, which might have triggered running julia code + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); Function *cw = Function::Create(FunctionType::get(sret ? T_void : prt, fargt_sig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(cw); cw->setAttributes(attrs); #ifdef LLVM37 @@ -3921,7 +3677,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v jl_throw(jl_memory_exception); list2->len = len; list2->data()[len-1].isref = isref; - list2->data()[len-1].f = imaging_mode ? cw : cw_proto; + list2->data()[len-1].f = cw_proto; lam->functionObjects.cFunctionList = list2; // See whether this function is specsig or jlcall @@ -4097,13 +3853,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v else builder.CreateRet(r); -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(cw); - FPM->run(*cw); -#endif - - cw->removeFromParent(); - active_module->getFunctionList().push_back(cw); + jl_finalize_module(std::unique_ptr(M)); // Restore the previous compile context if (old != NULL) { @@ -4118,7 +3868,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret) +static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret, Module *M) { std::stringstream funcName; const std::string &fname = f->getName().str(); @@ -4128,8 +3878,8 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct else funcName << fname; - Function *w = Function::Create(jl_func_sig, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(w); #ifdef LLVM37 w->addFnAttr("no-frame-pointer-elim", "true"); @@ -4198,10 +3948,9 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct } // Compile to LLVM IR, using a specialized signature if applicable. -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx) +static Module *emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations) { - assert(definitions && "Capturing definitions is always required"); + assert(declarations && "Capturing declarations is always required"); // step 1. unpack AST and allocate codegen context for this function jl_expr_t *ast = (jl_expr_t*)lam->ast; @@ -4227,7 +3976,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio ctx.vaStack = false; ctx.inbounds.push_back(false); ctx.boundsCheck.push_back(false); - ctx.cyclectx = cyclectx; ctx.spvals_ptr = NULL; // step 2. process var-info lists to see what vars need boxing @@ -4330,6 +4078,8 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio funcName << "_" << globalUnique++; ctx.sret = false; + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); if (specsig) { // assumes !va and !needsparams std::vector fsig(0); Type *rt; @@ -4355,37 +4105,28 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio fsig.push_back(ty); } f = Function::Create(FunctionType::get(rt, fsig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); if (ctx.sret) f->addAttribute(1, Attribute::StructRet); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->specFunctionObject = f; - if (declarations) - declarations->specFunctionObject = function_proto(f); - fwrap = gen_jlcall_wrapper(lam, ast, - (llvm::Function*)lam->functionObjects.specFunctionObject, ctx.sret); - definitions->functionObject = fwrap; - if (declarations) - declarations->functionObject = function_proto(fwrap); + fwrap = gen_jlcall_wrapper(lam, ast, f, ctx.sret, M); + declarations->functionObject = function_proto(fwrap); + declarations->specFunctionObject = function_proto(f); } else { f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->functionObject = f; - definitions->specFunctionObject = NULL; - if (declarations) { - declarations->functionObject = function_proto(f); - declarations->specFunctionObject = NULL; - } + declarations->functionObject = function_proto(f); + declarations->specFunctionObject = NULL; } if (jlrettype == (jl_value_t*)jl_bottom_type) f->setDoesNotReturn(); @@ -4443,7 +4184,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio filename = ""; int toplineno = lno; - DIBuilder dbuilder(*builtins_module); + DIBuilder dbuilder(*M); ctx.dbuilder = &dbuilder; #ifdef LLVM37 DIFile *topfile = NULL; @@ -4649,7 +4390,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio /* // step 6. (optional) check for stack overflow (the slower way) Value *cur_sp = - builder.CreateCall(Intrinsic::getDeclaration(jl_Module, + builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::frameaddress), ConstantInt::get(T_int32, 0)); Value *sp_ok = @@ -4681,12 +4422,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio // must be in the first basic block for the llvm mem2reg pass to work // get pointers for locals stored in the gc frame array (argTemp) -#ifdef LLVM36 - if (ctx.debug_enabled) { - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_declare)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_value)); - } -#endif for (std::map::iterator I = ctx.vars.begin(), E = ctx.vars.end(); I != E; ++I) { jl_sym_t *s = I->first; jl_varinfo_t &varinfo = I->second; @@ -5068,9 +4803,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio for(std::vector::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); InlineFunctionInfo info; - // Intrinsics that InlineFunction might create - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_start)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_end)); if (!InlineFunction(*it,info)) jl_error("Inlining Pass failed"); if (inlinef->getParent()) @@ -5081,26 +4813,14 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio } } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if (cyclectx) { - cyclectx->functions.push_back(f); - if (fwrap) - cyclectx->functions.push_back(fwrap); - } -#endif - // step 15. Perform any delayed instantiations if (ctx.debug_enabled) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if(cyclectx) - cyclectx->CUs.push_back(CU); -#endif ctx.dbuilder->finalize(); } JL_GC_POP(); - return; + return M; } // --- initialization --- @@ -5143,8 +4863,8 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs } else { // this assigns a function pointer (from loading the system image), to the function object - std::string funcName = jl_symbol_name(lam->name); - funcName = "julia_" + funcName; + std::stringstream funcName; + funcName << "jlsys_" << jl_symbol_name(lam->name) << "_" << globalUnique++; if (specsig) { // assumes !va std::vector fsig(0); jl_value_t *jlrettype = lam->rettype; @@ -5171,8 +4891,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs ty = PointerType::get(ty,0); fsig.push_back(ty); } - Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName, - shadow_module); + Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName.str(), shadow_output); if (sret) f->addAttribute(1, Attribute::StructRet); @@ -5187,7 +4906,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs lam->fptr = fptr; } else { - Function *f = jlcall_func_to_llvm(funcName, fptr, shadow_module); + Function *f = jlcall_func_to_llvm(funcName.str(), fptr, shadow_output); if (lam->functionObjects.functionObject == NULL) { lam->functionObjects.functionObject = (void*)f; assert(lam->fptr == NULL); @@ -5391,14 +5110,6 @@ static void init_julia_llvm_env(Module *m) NULL, "jl_dl_handle"); add_named_global(jldll_var, &jl_dl_handle); #endif -#ifdef JL_NEED_FLOATTEMP_VAR - // Has to be big enough for the biggest LLVM-supported float type - jlfloattemp_var = - addComdat(new GlobalVariable(*m, IntegerType::get(jl_LLVMContext,128), - false, GlobalVariable::ExternalLinkage, - ConstantInt::get(IntegerType::get(jl_LLVMContext,128),0), - "jl_float_temp")); -#endif #ifndef JULIA_ENABLE_THREADING // For non-threading, we use the address of the global variable directly @@ -5435,17 +5146,11 @@ static void init_julia_llvm_env(Module *m) PointerType *pfunctype = jltls_states_func->getFunctionType()->getPointerTo(); jltls_states_func_ptr = new GlobalVariable(*m, pfunctype, - false, GlobalVariable::InternalLinkage, - ConstantPointerNull::get(pfunctype), - "jl_get_ptls_states.ptr"); + false, GlobalVariable::ExternalLinkage, + NULL, "jl_get_ptls_states.ptr"); addComdat(jltls_states_func_ptr); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[jltls_states_func_ptr] = - (void*)jl_get_ptls_states_getter(); -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(jltls_states_func_ptr); + void **p = (void**)jl_emit_and_add_to_shadow(jltls_states_func_ptr); *p = (void*)jl_get_ptls_states_getter(); -#endif jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(jltls_states_func_ptr, T_psize)); jltls_states_func_idx = jl_sysimg_gvars.size(); @@ -5872,12 +5577,6 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlcall_frame_func, (void*)NULL, /*dllimport*/false); // set up optimization passes -#ifdef LLVM38 - FPM = new legacy::FunctionPassManager(m); -#else - FPM = new FunctionPassManager(m); -#endif - #ifdef LLVM37 // No DataLayout pass needed anymore. #elif defined(LLVM36) @@ -5888,11 +5587,22 @@ static void init_julia_llvm_env(Module *m) jl_data_layout = new DataLayout(*jl_ExecutionEngine->getDataLayout()); #endif +#ifndef USE_ORCJIT +#ifdef LLVM38 + PM = new legacy::PassManager(); +#else + PM = new PassManager(); +#endif +#ifndef LLVM37 + PM->add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); +#else + PM->add(new TargetLibraryInfoWrapperPass(Triple(jl_TargetMachine->getTargetTriple()))); +#endif #ifndef LLVM37 - FPM->add(jl_data_layout); + PM->add(jl_data_layout); +#endif + addOptimizationPasses(PM); #endif - addOptimizationPasses(FPM); - FPM->doInitialization(); } // Helper to figure out what features to set for the LLVM target @@ -5979,24 +5689,13 @@ extern "C" void jl_init_codegen(void) InitializeNativeTargetAsmParser(); Module *m, *engine_module; - + engine_module = new Module("julia", jl_LLVMContext); #ifdef USE_MCJIT - m = shadow_module = new Module("shadow", jl_LLVMContext); - builtins_module = new Module("julia_builtins", jl_LLVMContext); - if (imaging_mode) { - engine_module = new Module("engine_module", jl_LLVMContext); - active_module = shadow_module; - } - else { - active_module = new Module("julia", jl_LLVMContext); - engine_module = m; -#ifdef USE_ORCJIT - engine_module = new Module("engine_module", jl_LLVMContext); -#endif - } + m = new Module("julia", jl_LLVMContext); #else - engine_module = m = jl_Module = new Module("julia", jl_LLVMContext); + m = engine_module; #endif + shadow_output = m; TargetOptions options = TargetOptions(); //options.PrintMachineCode = true; //Print machine code produced during JIT compiling @@ -6098,19 +5797,9 @@ extern "C" void jl_init_codegen(void) mbuilder = new MDBuilder(getGlobalContext()); // Now that the execution engine exists, initialize all modules -#ifdef USE_MCJIT - jl_setup_module(engine_module); - jl_setup_module(shadow_module); - jl_setup_module(active_module); - jl_setup_module(builtins_module); -#else jl_setup_module(engine_module); -#endif - - if (imaging_mode) - init_julia_llvm_env(shadow_module); - else - init_julia_llvm_env(builtins_module); + jl_setup_module(m); + init_julia_llvm_env(m); #ifndef USE_ORCJIT jl_ExecutionEngine->RegisterJITEventListener(CreateJuliaJITEventListener()); diff --git a/src/codegen_internal.h b/src/codegen_internal.h index 02d08b8aa3034..860193849e59c 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -1,4 +1,6 @@ -// Pre-declaration. Definition in disasm.cpp +// This file is a part of Julia. License is MIT: http://julialang.org/license + +// Declarations for disasm.cpp extern "C" void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #ifndef USE_MCJIT @@ -12,6 +14,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #endif ); +// Declarations for debuginfo.cpp extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *section_slide, const object::ObjectFile **object, #ifdef USE_MCJIT @@ -20,8 +23,17 @@ extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int6 std::vector *lines #endif ); + extern bool jl_dylib_DI_for_fptr(size_t pointer, const object::ObjectFile **object, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename); + #ifdef USE_MCJIT extern void jl_cleanup_DI(llvm::DIContext *context); #endif + +#ifdef USE_ORCJIT +extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, + const object::ObjectFile &obj, + const object::ObjectFile &debugObj, + const RuntimeDyld::LoadedObjectInfo &L); +#endif diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 8acd29dac0f83..d28b399a9a932 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -238,6 +238,10 @@ class JuliaJITEventListener: public JITEventListener #endif FuncInfo tmp = {&F, Size, Details.LineStarts, linfo}; info[(size_t)(Code)] = tmp; +#ifndef KEEP_BODIES + if (!jl_generating_output()) + const_cast(&F)->deleteBody(); +#endif uv_rwlock_wrunlock(&threadsafe); } diff --git a/src/dump.c b/src/dump.c index a62366553cfc7..4b9c9cea88d8a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1921,7 +1921,7 @@ JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) } // Get handle to sys.so - jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL); + jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_LOCAL | JL_RTLD_NOW); // set cpu target if unspecified by user and available from sysimg // otherwise default to native. diff --git a/src/gf.c b/src/gf.c index 5d2286911d5d0..880c807f26d63 100644 --- a/src/gf.c +++ b/src/gf.c @@ -870,7 +870,7 @@ static jl_value_t *jl_call_unspecialized(jl_svec_t *sparam_vals, jl_lambda_info_ jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } assert(jl_svec_len(meth->sparam_syms) == jl_svec_len(sparam_vals)); @@ -1414,7 +1414,7 @@ jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_value_t *types, int lim); // compile-time method lookup -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) { assert(jl_nparams(types) > 0); if (!jl_is_leaf_type((jl_value_t*)types)) @@ -1451,7 +1451,7 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) if (sf->functionObjects.functionObject == NULL) { if (sf->fptr != NULL) goto not_found; - jl_compile_linfo(sf, cyclectx); + jl_compile_linfo(sf); } JL_GC_POP(); return sf; @@ -1506,7 +1506,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) if (jl_is_leaf_type((jl_value_t*)methsig)) { // usually can create a specialized version of the function, // if the signature is already a leaftype - jl_lambda_info_t *spec = jl_get_specialization1(methsig, NULL); + jl_lambda_info_t *spec = jl_get_specialization1(methsig); if (spec) { if (methsig == meth->sig) { // replace unspecialized func with newly specialized version @@ -1545,7 +1545,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) goto getnext; // signature wouldn't be callable / is invalid -- skip it } if (jl_is_leaf_type(sig)) { - if (jl_get_specialization1((jl_tupletype_t*)sig, NULL)) { + if (jl_get_specialization1((jl_tupletype_t*)sig)) { if (!jl_has_typevars((jl_value_t*)sig)) goto getnext; // success } } @@ -1680,7 +1680,7 @@ static void _compile_all_deq(jl_array_t *found) linfo->functionID = -1; } else { - jl_compile_linfo(linfo, NULL); + jl_compile_linfo(linfo); assert(linfo->functionID > 0); } } @@ -1783,7 +1783,7 @@ void jl_compile_all(void) JL_DLLEXPORT void jl_compile_hint(jl_tupletype_t *types) { - (void)jl_get_specialization1(types, NULL); + (void)jl_get_specialization1(types); } #ifdef JL_TRACE diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index cb68f82a3f6ad..573e127e3bc90 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1012,8 +1012,9 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, // understood that everything is implicitly rounded to 23 bits, // but if we start looking at more bits we need to actually do the // rounding first instead of carrying around incorrect low bits. - builder.CreateStore(FP(x), builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); - x = builder.CreateLoad(builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); + Value *jlfloattemp_var = emit_static_alloca(FT(x->getType())); + builder.CreateStore(FP(x), jlfloattemp_var); + x = builder.CreateLoad(jlfloattemp_var, true); #endif return mark_julia_type(builder.CreateFPExt(x, FTnbits(nb)), false, bt, ctx); } @@ -1140,12 +1141,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, case fma_float: { assert(y->getType() == x->getType()); assert(z->getType() == y->getType()); - Value *fmaintr = Intrinsic::getDeclaration(builtins_module, Intrinsic::fma, + Value *fmaintr = Intrinsic::getDeclaration(jl_Module, Intrinsic::fma, ArrayRef(x->getType())); #ifdef LLVM37 - return builder.CreateCall(prepare_call(fmaintr),{ FP(x), FP(y), FP(z) }); + return builder.CreateCall(fmaintr,{ FP(x), FP(y), FP(z) }); #else - return builder.CreateCall3(prepare_call(fmaintr), FP(x), FP(y), FP(z)); + return builder.CreateCall3(fmaintr, FP(x), FP(y), FP(z)); #endif } case muladd_float: @@ -1158,8 +1159,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, #else return builder.CreateCall3 #endif - (prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::fmuladd, - ArrayRef(x->getType()))), + (Intrinsic::getDeclaration(jl_Module, Intrinsic::fmuladd, + ArrayRef(x->getType())), #ifdef LLVM37 {FP(x), FP(y), FP(z)} #else @@ -1181,7 +1182,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *ix = JL_INT(x); Value *iy = JL_INT(y); assert(ix->getType() == iy->getType()); Value *intr = - Intrinsic::getDeclaration(builtins_module, + Intrinsic::getDeclaration(jl_Module, f==checked_sadd_int ? Intrinsic::sadd_with_overflow : (f==checked_uadd_int ? @@ -1195,9 +1196,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Intrinsic::umul_with_overflow)))), ArrayRef(ix->getType())); #ifdef LLVM37 - Value *res = builder.CreateCall(prepare_call(intr),{ix, iy}); + Value *res = builder.CreateCall(intr,{ix, iy}); #else - Value *res = builder.CreateCall2(prepare_call(intr), ix, iy); + Value *res = builder.CreateCall2(intr, ix, iy); #endif Value *obit = builder.CreateExtractValue(res, ArrayRef(1)); raise_exception_if(obit, prepare_global(jlovferr_var), ctx); @@ -1335,33 +1336,33 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateAShr(x, uint_cnvt(t,y))); case bswap_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::bswap, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::bswap, + ArrayRef(x->getType())), x); case ctpop_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::ctpop, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::ctpop, + ArrayRef(x->getType())), x); case ctlz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *ctlz = Intrinsic::getDeclaration(builtins_module, Intrinsic::ctlz, + Value *ctlz = Intrinsic::getDeclaration(jl_Module, Intrinsic::ctlz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(ctlz), {x, ConstantInt::get(T_int1,0)}); + return builder.CreateCall(ctlz, {x, ConstantInt::get(T_int1,0)}); #else - return builder.CreateCall2(prepare_call(ctlz), x, ConstantInt::get(T_int1,0)); + return builder.CreateCall2(ctlz, x, ConstantInt::get(T_int1,0)); #endif } case cttz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *cttz = Intrinsic::getDeclaration(builtins_module, Intrinsic::cttz, ArrayRef(types)); + Value *cttz = Intrinsic::getDeclaration(jl_Module, Intrinsic::cttz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(cttz), {x, ConstantInt::get(T_int1, 0)}); + return builder.CreateCall(cttz, {x, ConstantInt::get(T_int1, 0)}); #else - return builder.CreateCall2(prepare_call(cttz), x, ConstantInt::get(T_int1, 0)); + return builder.CreateCall2(cttz, x, ConstantInt::get(T_int1, 0)); #endif } @@ -1378,9 +1379,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, { x = FP(x); #ifdef LLVM34 - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::fabs, - ArrayRef(x->getType()))), + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::fabs, + ArrayRef(x->getType())), x); #else Type *intt = JL_INTT(x->getType()); @@ -1431,34 +1432,34 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case ceil_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::ceil, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::ceil, + ArrayRef(x->getType())), x); } case floor_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::floor, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::floor, + ArrayRef(x->getType())), x); } case trunc_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::trunc, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::trunc, + ArrayRef(x->getType())), x); } case rint_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::rint, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::rint, + ArrayRef(x->getType())), x); } case sqrt_llvm: { x = FP(x); raise_exception_unless(builder.CreateFCmpUGE(x, ConstantFP::get(x->getType(),0.0)), prepare_global(jldomerr_var), ctx); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } case powi_llvm: { @@ -1467,12 +1468,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Type *tx = x->getType(); // TODO: LLVM expects this to be i32 #ifdef LLVM36 Type *ts[1] = { tx }; - Value *powi = Intrinsic::getDeclaration(builtins_module, Intrinsic::powi, + Value *powi = Intrinsic::getDeclaration(jl_Module, Intrinsic::powi, ArrayRef(ts)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(powi), {x, y}); + return builder.CreateCall(powi, {x, y}); #else - return builder.CreateCall2(prepare_call(powi), x, y); + return builder.CreateCall2(powi, x, y); #endif #else // issue #6506 @@ -1482,8 +1483,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case sqrt_llvm_fast: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 78a56b3bfe9a2..400d49fb01d8f 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,5 +1,6 @@ // This file is part of Julia. -// Parts of this file are copied from LLVM, under the UIUC license. + +// Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). template static void addOptimizationPasses(T *PM) @@ -158,11 +159,6 @@ extern "C" { LLVM_ATTRIBUTE_NOINLINE extern void __jit_debug_register_code(); } -extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, - const object::ObjectFile &obj, - const object::ObjectFile &debugObj, - const RuntimeDyld::LoadedObjectInfo &L); - #if defined(_OS_DARWIN_) && defined(LLVM37) && defined(LLVM_SHLIB) #define CUSTOM_MEMORY_MANAGER createRTDyldMemoryManagerOSX extern RTDyldMemoryManager *createRTDyldMemoryManagerOSX(); @@ -313,23 +309,47 @@ class JuliaOJIT { report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr); } - std::string mangle(const std::string &Name) + std::string getMangledName(const std::string &Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return MangledName; + SmallString<128> FullName; + Mangler::getNameWithPrefix(FullName, Name, DL); + return FullName.str(); + } + + std::string getMangledName(const GlobalValue *GV) + { + return getMangledName(GV->getName()); } - void addGlobalMapping(StringRef Name, void *Addr) + void addGlobalMapping(StringRef Name, uint64_t Addr) { - GlobalSymbolTable[mangle(Name)] = Addr; + bool successful = GlobalSymbolTable.insert(make_pair(getMangledName(Name), (void*)Addr)).second; + assert(successful); } - ModuleHandleT addModule(Module *M) + void *getPointerToGlobalIfAvailable(StringRef S) { + GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); + if (pos != GlobalSymbolTable.end()) + return pos->second; + return nullptr; + } + + ModuleHandleT addModule(std::unique_ptr M) { +#ifndef NDEBUG + // validate the relocations for M + for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { + Function *F = &*I; + ++I; + if (F->isDeclaration()) { + if (F->use_empty()) + F->eraseFromParent(); + else + assert(F->isIntrinsic() || findUnmangledSymbol(F->getName()) || + SectionMemoryManager::getSymbolAddressInProcess(F->getName())); + } + } +#endif // We need a memory manager to allocate memory and resolve symbols for this // new module. Create one that resolves symbols by looking back into the // JIT. @@ -350,7 +370,7 @@ class JuliaOJIT { [](const std::string &S) { return nullptr; } ); SmallVector,1> Ms; - Ms.push_back(std::unique_ptr{M}); + Ms.push_back(std::move(M)); return CompileLayer->addModuleSet(std::move(Ms), MemMgr, std::move(Resolver)); @@ -362,9 +382,9 @@ class JuliaOJIT { { if (ExportedSymbolsOnly) { // Step 1: Check against list of known external globals - GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(Name); - if (pos != GlobalSymbolTable.end()) - return orc::JITSymbol((uintptr_t)pos->second, JITSymbolFlags::Exported); + void *Addr = getPointerToGlobalIfAvailable(Name); + if (Addr != nullptr) + return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } // Step 2: Search all previously emitted symbols return CompileLayer->findSymbol(Name, ExportedSymbolsOnly); @@ -372,17 +392,17 @@ class JuliaOJIT { orc::JITSymbol findUnmangledSymbol(const std::string Name) { - return findSymbol(mangle(Name), true); + return findSymbol(getMangledName(Name), true); } uint64_t getGlobalValueAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } uint64_t getFunctionAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } Function *FindFunctionNamed(const std::string &Name) @@ -422,3 +442,528 @@ class JuliaOJIT { } #endif + +#ifdef USE_ORCJIT +JuliaOJIT *jl_ExecutionEngine; +#else +ExecutionEngine *jl_ExecutionEngine; +#endif + +// destructively move the contents of src into dest +// this assumes that the targets of the two modules are the same +// including the DataLayout and ModuleFlags (for example) +// and that there is no module-level assembly +static void jl_merge_module(Module *dest, std::unique_ptr src) +{ + for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) { + GlobalVariable *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getGlobalList().push_back(sG); + } + + for (Module::iterator I = src->begin(), E = src->end(); I != E;) { + Function *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getFunctionList().push_back(sG); + } + + for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) { + GlobalAlias *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getAliasList().push_back(sG); + } + +// for (Module::named_metadata_iterator I = src->named_metadata_begin(), E = src->named_metadata_end(); I != E;) { +// NamedMDNode *sG = &*I; +// NamedMDNode *dG = dest->getOrInsertNamedMetadata(sG->getName()); +// ++I; +// for (NamedMDNode::op_iterator OpI = sG->op_begin(), OpE = sG->op_end(); OpI != OpE; ++OpI) { +// dG->addOperand(*OpI); +// } +// sG->eraseFromParent(); +// } +#ifdef LLVM35 + NamedMDNode *sNMD = src->getNamedMetadata("llvm.dbg.cu"); + if (sNMD) { + NamedMDNode *dNMD = dest->getOrInsertNamedMetadata("llvm.dbg.cu"); + for (NamedMDNode::op_iterator I = sNMD->op_begin(), E = sNMD->op_end(); I != E; ++I) { + dNMD->addOperand(*I); + } + } +#endif +} + +#ifdef USE_MCJIT +static StringMap module_for_fname; +static bool jl_finalize_function(Function *F, Module *collector = NULL) +{ + std::unique_ptr m(module_for_fname.lookup(F->getName())); + if (m) { + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + module_for_fname.erase(F->getName()); + } + } + + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + if (F->isDeclaration() && !F->isIntrinsic()) { + bool modified = jl_finalize_function(&*I, collector ? collector : m.get()); + if (modified && !collector) { + // jl_finalize_function modifies the collector, + // so need to restart if this was iterating over it + I = m->begin(); + E = m->end(); + } + } + } + + if (collector) { + jl_merge_module(collector, std::move(m)); + } + else { +#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(USE_MCJIT) + ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); +#endif + assert(jl_ExecutionEngine); +#if defined(LLVM36) + jl_ExecutionEngine->addModule(std::move(m)); +#else + jl_ExecutionEngine->addModule(m.release()); +#endif + } + return true; + } + return false; +} +#endif + +template // for GlobalObject's +static T *addComdat(T *G) +{ +#if defined(_OS_WINDOWS_) + if (imaging_mode && !G->isDeclaration()) { +#ifdef LLVM35 + // Add comdat information to make MSVC link.exe happy + Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); + jl_Comdat->setSelectionKind(Comdat::NoDuplicates); + G->setComdat(jl_Comdat); + // add __declspec(dllexport) to everything marked for export + if (G->getLinkage() == GlobalValue::ExternalLinkage) + G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); +#endif + } +#endif + return G; +} + +// Connect Modules via prototypes + +static GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) +{ + GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), + G->isConstant(), GlobalVariable::ExternalLinkage, + NULL, G->getName(), G->getThreadLocalMode()); + if (M) + M->getGlobalList().push_back(proto); + return proto; +} + +static Function *function_proto(Function *F, Module *M = NULL) +{ + Function *NewF = Function::Create(F->getFunctionType(), + Function::ExternalLinkage, + F->getName(), M); + NewF->setAttributes(AttributeSet()); + + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + +#ifdef LLVM37 + // Declarations are not allowed to have personality routines, but + // copyAttributesFrom sets them anyway, so clear them again manually + NewF->setPersonalityFn(nullptr); +#endif + + return NewF; +} + +template +#ifdef LLVM35 +static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) +#else +static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) +#endif +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + void *addr = (void*)(uintptr_t)_addr; +#ifdef LLVM34 + StringRef name = gv->getName(); +#ifdef _OS_WINDOWS_ + std::string imp_name; +#endif +#endif + +#ifdef _OS_WINDOWS_ + // setting JL_DLLEXPORT correctly only matters when building a binary + if (dllimport && imaging_mode) { + assert(gv->getLinkage() == GlobalValue::ExternalLinkage); +#ifdef LLVM35 + // add the __declspec(dllimport) attribute + gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); + // this will cause llvm to rename it, so we do the same + imp_name = Twine("__imp_", name).str(); + name = StringRef(imp_name); +#else + gv->setLinkage(GlobalValue::DLLImportLinkage); +#endif +#if defined(_P64) || defined(LLVM35) + // __imp_ variables are indirection pointers, so use malloc to simulate that + void **imp_addr = (void**)malloc(sizeof(void**)); + *imp_addr = addr; + addr = (void*)imp_addr; +#endif + } +#endif // _OS_WINDOWS_ + +#ifdef USE_ORCJIT + addComdat(gv); + jl_ExecutionEngine->addGlobalMapping(name, (uintptr_t)addr); +#elif defined(USE_MCJIT) + addComdat(gv); + sys::DynamicLibrary::AddSymbol(name, addr); +#else // USE_MCJIT + jl_ExecutionEngine->addGlobalMapping(gv, addr); +#endif // USE_MCJIT +} + +static std::vector jl_sysimg_gvars; +static std::vector jl_sysimg_fvars; +typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing +static std::map jl_value_to_llvm; + +static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL) +{ + PointerType *T = cast(gv->getType()->getElementType()); // pointer is the only supported type currently + + GlobalVariable *shadowvar = NULL; +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + if (imaging_mode) +#endif + shadowvar = global_proto(gv, shadow_output); + + if (shadowvar) { + shadowvar->setInitializer(ConstantPointerNull::get(T)); + shadowvar->setLinkage(GlobalVariable::InternalLinkage); + addComdat(shadowvar); + if (imaging_mode && gvarinit) { + // make the pointer valid for future sessions + jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(shadowvar, T_psize)); + jl_value_llvm gv_struct; + gv_struct.gv = global_proto(gv); + gv_struct.index = jl_sysimg_gvars.size(); + jl_value_to_llvm[gvarinit] = gv_struct; + } + } + + // make the pointer valid for this session +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *slot = calloc(1, sizeof(void*)); + jl_ExecutionEngine->addGlobalMapping(gv->getName(), (uintptr_t)slot); + return slot; +#else + return jl_ExecutionEngine->getPointerToGlobal(shadowvar); +#endif +} + +static void* jl_get_global(GlobalVariable *gv) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(gv)); +#else + void *p = jl_ExecutionEngine->getPointerToGlobal( + shadow_output->getNamedValue(gv->getName())); +#endif + assert(p); + return p; +} + +static void jl_add_to_shadow(Module *m) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + if (!imaging_mode) + return; + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(m, VMap)); + for (Module::iterator I = clone->begin(), E = clone->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + F->setLinkage(Function::InternalLinkage); + addComdat(F); + } + } +#else + // on the old jit, the shadow_module is the same as the execution engine_module + std::unique_ptr clone(m); +#endif + jl_merge_module(shadow_output, std::move(clone)); +} + +#ifdef HAVE_CPUID +extern "C" { + extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); +} +#endif + +static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, + const char *sysimg_data, size_t sysimg_len) +{ + ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); + addComdat(new GlobalVariable(*mod, + gvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), + "jl_sysimg_gvars")); + ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); + addComdat(new GlobalVariable(*mod, + fvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), + "jl_sysimg_fvars")); + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, globalUnique+1), + "jl_globalUnique")); +#ifdef JULIA_ENABLE_THREADING + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, jltls_states_func_idx), + "jl_ptls_states_getter_idx")); +#endif + + Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); + addComdat(new GlobalVariable(*mod, + feature_string->getType(), + true, + GlobalVariable::ExternalLinkage, + feature_string, + "jl_sysimg_cpu_target")); + +#ifdef HAVE_CPUID + // For native also store the cpuid + if (strcmp(jl_options.cpu_target,"native") == 0) { + uint32_t info[4]; + + jl_cpuid((int32_t*)info, 1); + addComdat(new GlobalVariable(*mod, + T_int64, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + "jl_sysimg_cpu_cpuid")); + } +#endif + + if (sysimg_data) { + Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); + addComdat(new GlobalVariable(*mod, data->getType(), true, + GlobalVariable::ExternalLinkage, + data, "jl_system_image_data")); + Constant *len = ConstantInt::get(T_size, sysimg_len); + addComdat(new GlobalVariable(*mod, len->getType(), true, + GlobalVariable::ExternalLinkage, + len, "jl_system_image_size")); + } +} + +static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, + bool dump_as_bc) +{ +#ifdef JL_DEBUG_BUILD + verifyModule(*shadow_output); +#endif + +#ifdef LLVM36 + std::error_code err; + StringRef fname_ref = StringRef(fname); + raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); +#elif defined(LLVM35) + std::string err; + raw_fd_ostream OS(fname, err, sys::fs::F_None); +#else + std::string err; + raw_fd_ostream OS(fname, err); +#endif +#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone + raw_fd_ostream& FOS(OS); +#else + formatted_raw_ostream FOS(OS); +#endif + + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); +#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::COFF); +#else + TheTriple.setEnvironment(Triple::UnknownEnvironment); +#endif +#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::MachO); +#else + TheTriple.setEnvironment(Triple::MachO); +#endif +#endif +#ifdef LLVM35 + std::unique_ptr +#else + OwningPtr +#endif + TM(jl_TargetMachine->getTarget().createTargetMachine( + TheTriple.getTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + Reloc::PIC_, +#else + jit_model ? Reloc::PIC_ : Reloc::Default, +#endif + jit_model ? CodeModel::JITDefault : CodeModel::Default, + CodeGenOpt::Aggressive // -O3 + )); + +#ifdef LLVM38 + legacy::PassManager PM; +#else + PassManager PM; +#endif +#ifndef LLVM37 + PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); +#else + PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); +#endif + + + // set up optimization passes +#ifdef LLVM37 + // No DataLayout pass needed anymore. +#elif defined(LLVM36) + PM.add(new DataLayoutPass()); +#elif defined(LLVM35) + PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); +#else + PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); +#endif + + addOptimizationPasses(&PM); + if (!dump_as_bc) { + if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { + jl_error("Could not generate obj file for this target"); + } + } + + // now copy the module, since PM.run may modify it + ValueToValueMapTy VMap; +#ifdef LLVM38 + Module *clone = CloneModule(shadow_output, VMap).release(); +#else + Module *clone = CloneModule(shadow_output, VMap); +#endif +#ifdef LLVM37 + // Reset the target triple to make sure it matches the new target machine + clone->setTargetTriple(TM->getTargetTriple().str()); +#ifdef LLVM38 + clone->setDataLayout(TM->createDataLayout()); +#else + clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); +#endif +#endif + + // add metadata information + jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); + + // do the actual work + PM.run(*clone); + if (dump_as_bc) + WriteBitcodeToFile(clone, FOS); + delete clone; +} + +static int32_t jl_assign_functionID(Function *functionObject, int specsig) +{ + // give the function an index in the constant lookup table + if (!imaging_mode) + return 0; + jl_sysimg_fvars.push_back(ConstantExpr::getBitCast( + shadow_output->getNamedValue(functionObject->getName()), + T_pvoidfunc)); + return jl_sysimg_fvars.size(); +} + +extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) +{ + // map a jl_value_t memory location to a GlobalVariable + std::map::iterator it; + it = jl_value_to_llvm.find(p); + if (it == jl_value_to_llvm.end()) + return 0; + return it->second.index; +} diff --git a/src/julia.h b/src/julia.h index d293159bca69b..367d7d2a6b6b8 100644 --- a/src/julia.h +++ b/src/julia.h @@ -211,6 +211,7 @@ typedef struct _jl_lambda_info_t { // with the same name as the generated functions for this linfo, suitable // for referencing in LLVM IR jl_llvm_functions_t functionObjects; + void* moduleObject; // llvm::Module* containing definition int32_t functionID; // index that this function will have in the codegen table int32_t specFunctionID; // index that this specFunction will have in the codegen table diff --git a/src/julia_internal.h b/src/julia_internal.h index 4a50cf2e682e6..72edff4c96b54 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -56,13 +56,13 @@ STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) } void jl_generate_fptr(jl_lambda_info_t *li); -void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx); +void jl_compile_linfo(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } if (meth->jlcall_api == 0) @@ -263,7 +263,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); void jl_idtable_rehash(jl_array_t **pa, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx); +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); jl_function_t *jl_module_get_initializer(jl_module_t *m); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); diff --git a/src/toplevel.c b/src/toplevel.c index 1eb3fe14742ac..44a010370fc5b 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -81,7 +81,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); - jl_get_specialization1((jl_tupletype_t*)tt, NULL); + jl_get_specialization1((jl_tupletype_t*)tt); JL_GC_POP(); } } diff --git a/test/llvmcall.jl b/test/llvmcall.jl index 90e5ccc6d3d29..2c58958d01237 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -70,19 +70,12 @@ end Int32(1), Int32(2))) == 3 # Test whether declarations work properly -# This test only work properly for llvm 3.5+ -# On llvm <3.5+ even though the the compilation fails on the first try, -# llvm still adds the intrinsice declaration to the module and subsequent calls -# are succesfull. -#if convert(VersionNumber, Base.libllvm_version) > v"3.5-" -# -#function undeclared_ceil(x::Float64) -# llvmcall("""%2 = call double @llvm.ceil.f64(double %0) -# ret double %2""", Float64, Tuple{Float64}, x) -#end -#@test_throws ErrorException undeclared_ceil(4.2) -# -#end +function undeclared_ceil(x::Float64) + llvmcall("""%2 = call double @llvm.ceil.f64(double %0) + ret double %2""", Float64, Tuple{Float64}, x) +end +@test_throws ErrorException undeclared_ceil(4.2) +@test_throws ErrorException undeclared_ceil(4.2) function declared_floor(x::Float64) llvmcall(