llvmcall: use AlwaysInliner + test inlining behavior

src/ccall.cpp

@@ -1000,6 +1000,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
  assert(isPtr);
  // Create Function skeleton
  f = (llvm::Function*)jl_unbox_voidpointer(ir);
+ assert(!f->isDeclaration());
  assert(f->getReturnType() == rettype);
  int i = 0;
  for (std::vector<Type *>::iterator it = argtypes.begin();
@@ -1025,15 +1026,6 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
  }
  }
 
- /*
- * It might be tempting to just try to set the Always inline attribute on the function
- * and hope for the best. However, this doesn't work since that would require an inlining
- * pass (which is a Call Graph pass and cannot be managed by a FunctionPassManager). Instead
- * We are sneaky and call the inliner directly. This however doesn't work until we've actually
- * generated the entire function, so we need to store it in the context until the end of the
- * function. This also has the benefit of looking exactly like we cut/pasted it in in `code_llvm`.
- */
-
  // Since we dumped all of f's dependencies into the active module,
  // we cannot reasonably inline it, so leave it there and just emit
  // a regular call
@@ -1064,7 +1056,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
  mark_gc_uses(gc_uses);
  CallInst *inst = builder.CreateCall(f, ArrayRef<Value*>(&argvals[0], nargt));
  if (isString)
- ctx->to_inline.push_back(inst);
+ f->addFnAttr(Attribute::AlwaysInline);
  mark_gc_uses(gc_uses);
 
  JL_GC_POP();

src/codegen.cpp

@@ -548,7 +548,6 @@ typedef struct {
 
  bool debug_enabled;
  bool is_inbounds{false};
- std::vector<CallInst*> to_inline;
 } jl_codectx_t;
 
 static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx);
@@ -5025,22 +5024,7 @@ static std::unique_ptr<Module> emit_function(jl_method_instance_t *lam, jl_code_
  builder.SetCurrentDebugLocation(noDbg);
  builder.ClearInsertionPoint();
 
- // step 13, Apply LLVM level inlining
- for(std::vector<CallInst*>::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) {
- Function *inlinef = (*it)->getCalledFunction();
- assert(inlinef->getParent());
- InlineFunctionInfo info;
- if (!InlineFunction(*it,info))
- jl_error("Inlining Pass failed");
- if (inlinef->getParent())
- inlinef->eraseFromParent();
- else {
- inlinef->dropAllReferences();
- delete inlinef;
- }
- }
-
- // step 14. Perform any delayed instantiations
+ // step 13. Perform any delayed instantiations
  if (ctx.debug_enabled) {
  dbuilder.finalize();
  }

src/jitlayers.cpp

@@ -27,6 +27,7 @@
 #include <polly/CodeGen/CodegenCleanup.h>
 #endif
 
+#include <llvm/Transforms/IPO.h>
 #include <llvm/Transforms/Scalar.h>
 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
 #include <llvm/Transforms/Instrumentation.h>
@@ -141,6 +142,7 @@ void addOptimizationPasses(PassManager *PM)
  // list of passes from vmkit
  PM->add(createCFGSimplificationPass()); // Clean up disgusting code
  PM->add(createPromoteMemoryToRegisterPass());// Kill useless allocas
+ PM->add(createAlwaysInlinerPass()); // Respect always_inline
 
 #ifndef INSTCOMBINE_BUG
  PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl.

test/llvmcall.jl

@@ -85,6 +85,8 @@ function declared_floor(x::Float64)
  Float64, Tuple{Float64}, x)
 end
 @test declared_floor(4.2) ≈ 4.
+ir = sprint(io->code_llvm(io, declared_floor, Tuple{Float64}))
+@test contains(ir, "call double @llvm.floor.f64") # should be inlined
 
 function doubly_declared_floor(x::Float64)
  llvmcall(