diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index c0931e82d9875a..d714af035d21a2 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -1192,6 +1192,7 @@ struct PushAndPopStackRAII { CodeGenFunction::JumpDest Dest = CGF.getOMPCancelDestination(OMPD_parallel); CGF.EmitBranchThroughCleanup(Dest); + return llvm::Error::success(); }; // TODO: Remove this once we emit parallel regions through the @@ -2331,8 +2332,11 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, auto *OMPRegionInfo = dyn_cast_or_null(CGF.CapturedStmtInfo); if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { - CGF.Builder.restoreIP(OMPBuilder.createBarrier( - CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall, + EmitChecks); + assert(AfterIP && "unexpected error creating barrier"); + CGF.Builder.restoreIP(*AfterIP); return; } @@ -5928,8 +5932,10 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); }; - OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction, - IsOffloadEntry, OutlinedFn, OutlinedFnID); + llvm::Error Err = OMPBuilder.emitTargetRegionFunction( + EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn, + OutlinedFnID); + assert(!Err && "unexpected error creating target region"); if (!OutlinedFn) return; @@ -9670,9 +9676,12 @@ static void emitTargetCallKernelLaunch( NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads, DynCGGroupMem, HasNoWait); - CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().emitKernelLaunch( - CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID, - RTLoc, AllocaIP)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPRuntime->getOMPBuilder().emitKernelLaunch( + CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID, + RTLoc, AllocaIP); + assert(AfterIP && "unexpected error creating kernel launch"); + CGF.Builder.restoreIP(*AfterIP); }; if (RequiresOuterTask) @@ -10349,9 +10358,12 @@ void CGOpenMPRuntime::emitTargetDataCalls( InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(), CGF.Builder.GetInsertPoint()); llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP); - CGF.Builder.restoreIP(OMPBuilder.createTargetData( - OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB, - /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createTargetData( + OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB, + /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc); + assert(AfterIP && "unexpected error creating target data"); + CGF.Builder.restoreIP(*AfterIP); } void CGOpenMPRuntime::emitTargetDataStandAloneCall( diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index c66d5d11b0bbfa..598b946ad88dbb 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1753,11 +1753,14 @@ void CGOpenMPRuntimeGPU::emitReduction( Idx++; } - CGF.Builder.restoreIP(OMPBuilder.createReductionsGPU( - OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction, - DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang, - CGF.getTarget().getGridValue(), C.getLangOpts().OpenMPCUDAReductionBufNum, - RTLoc)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createReductionsGPU( + OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction, + DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang, + CGF.getTarget().getGridValue(), + C.getLangOpts().OpenMPCUDAReductionBufNum, RTLoc); + assert(AfterIP && "unexpected error creating GPU reductions"); + CGF.Builder.restoreIP(*AfterIP); return; } diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index 0da7855ab05c6c..1c32a675380c7f 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -1809,6 +1809,7 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { // thus calls destructors etc. auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; // Privatization callback that performs appropriate action for @@ -1831,15 +1832,18 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody( *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel"); + return llvm::Error::success(); }; CGCapturedStmtInfo CGSI(*CS, CR_OpenMP); CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI); llvm::OpenMPIRBuilder::InsertPointTy AllocaIP( AllocaInsertPt->getParent(), AllocaInsertPt->getIterator()); - Builder.restoreIP( + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB, - IfCond, NumThreads, ProcBind, S.hasCancel())); + IfCond, NumThreads, ProcBind, S.hasCancel()); + assert(AfterIP && "unexpected error creating parallel"); + Builder.restoreIP(*AfterIP); return; } @@ -2128,9 +2132,13 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { RunCleanupsScope BodyScope(*this); EmitStmt(BodyStmt); + return llvm::Error::success(); }; - llvm::CanonicalLoopInfo *CL = + + llvm::Expected Result = OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal); + assert(Result && "unexpected error creating canonical loop"); + llvm::CanonicalLoopInfo *CL = *Result; // Finish up the loop. Builder.restoreIP(CL->getAfterIP()); @@ -4016,11 +4024,13 @@ static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CGM.getOpenMPRuntime().getOMPBuilder(); llvm::OpenMPIRBuilder::InsertPointTy AllocaIP( CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator()); - OMPBuilder.applyWorkshareLoop( - CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier, - SchedKind, ChunkSize, /*HasSimdModifier=*/false, - /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false, - /*HasOrderedClause=*/false); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.applyWorkshareLoop( + CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, + NeedsBarrier, SchedKind, ChunkSize, /*HasSimdModifier=*/false, + /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false, + /*HasOrderedClause=*/false); + assert(AfterIP && "unexpected error creating workshare loop"); return; } @@ -4257,6 +4267,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; const CapturedStmt *ICS = S.getInnermostCapturedStmt(); @@ -4269,6 +4280,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, SubStmt, AllocaIP, CodeGenIP, "section"); + return llvm::Error::success(); }; SectionCBVector.push_back(SectionCB); } @@ -4277,6 +4289,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, CapturedStmt, AllocaIP, CodeGenIP, "section"); + return llvm::Error::success(); }; SectionCBVector.push_back(SectionCB); } @@ -4298,9 +4311,12 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI); llvm::OpenMPIRBuilder::InsertPointTy AllocaIP( AllocaInsertPt->getParent(), AllocaInsertPt->getIterator()); - Builder.restoreIP(OMPBuilder.createSections( - Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(), - S.getSingleClause())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createSections(Builder, AllocaIP, SectionCBVector, PrivCB, + FiniCB, S.hasCancel(), + S.getSingleClause()); + assert(AfterIP && "unexpected error creating sections"); + Builder.restoreIP(*AfterIP); return; } { @@ -4326,17 +4342,22 @@ void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt(); auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section"); + return llvm::Error::success(); }; LexicalScope Scope(*this, S.getSourceRange()); EmitStopPoint(&S); - Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createSection(Builder, BodyGenCB, FiniCB); + assert(AfterIP && "unexpected error creating section"); + Builder.restoreIP(*AfterIP); return; } @@ -4407,17 +4428,22 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master"); + return llvm::Error::success(); }; LexicalScope Scope(*this, S.getSourceRange()); EmitStopPoint(&S); - Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB); + assert(AfterIP && "unexpected error creating master"); + Builder.restoreIP(*AfterIP); return; } @@ -4453,18 +4479,22 @@ void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) { auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked"); + return llvm::Error::success(); }; LexicalScope Scope(*this, S.getSourceRange()); EmitStopPoint(&S); - Builder.restoreIP( - OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal); + assert(AfterIP && "unexpected error creating masked"); + Builder.restoreIP(*AfterIP); return; } @@ -4493,19 +4523,23 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical"); + return llvm::Error::success(); }; LexicalScope Scope(*this, S.getSourceRange()); EmitStopPoint(&S); - Builder.restoreIP(OMPBuilder.createCritical( - Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(), - HintInst)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB, + S.getDirectiveName().getAsString(), HintInst); + assert(AfterIP && "unexpected error creating critical"); + Builder.restoreIP(*AfterIP); return; } @@ -5464,11 +5498,15 @@ void CodeGenFunction::EmitOMPTaskgroupDirective( InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); + return llvm::Error::success(); }; CodeGenFunction::CGCapturedStmtInfo CapStmtInfo; if (!CapturedStmtInfo) CapturedStmtInfo = &CapStmtInfo; - Builder.restoreIP(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB); + assert(AfterIP && "unexpected error creating taskgroup"); + Builder.restoreIP(*AfterIP); return; } auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { @@ -6041,6 +6079,7 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); + return llvm::Error::success(); }; auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP, @@ -6064,11 +6103,14 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { OMPBuilderCBHelpers::EmitOMPInlinedRegionBody( *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered"); } + return llvm::Error::success(); }; OMPLexicalScope Scope(*this, S, OMPD_unknown); - Builder.restoreIP( - OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C); + assert(AfterIP && "unexpected error creating ordered"); + Builder.restoreIP(*AfterIP); } return; } @@ -7344,8 +7386,10 @@ void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { if (IfCond) IfCondition = EmitScalarExpr(IfCond, /*IgnoreResultAssign=*/true); - return Builder.restoreIP( - OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()); + assert(AfterIP && "unexpected error creating cancel"); + return Builder.restoreIP(*AfterIP); } } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index c4735ec41e7134..3afb9d84278e81 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -520,6 +520,9 @@ class OpenMPIRBuilder { /// Type used throughout for insertion points. using InsertPointTy = IRBuilder<>::InsertPoint; + /// Type used to represent an insertion point or an error value. + using InsertPointOrErrorTy = Expected; + /// Get the create a name using the platform specific separators. /// \param Parts parts of the final name that needs separation /// The created name has a first separator between the first and second part @@ -538,7 +541,7 @@ class OpenMPIRBuilder { /// A finalize callback knows about all objects that need finalization, e.g. /// destruction, when the scope of the currently generated construct is left /// at the time, and location, the callback is invoked. - using FinalizeCallbackTy = std::function; + using FinalizeCallbackTy = std::function; struct FinalizationInfo { /// The finalization callback provided by the last in-flight invocation of @@ -589,15 +592,19 @@ class OpenMPIRBuilder { /// not be split. /// \param CodeGenIP is the insertion point at which the body code should be /// placed. + /// + /// \return an error, if any were triggered during execution. using BodyGenCallbackTy = - function_ref; + function_ref; // This is created primarily for sections construct as llvm::function_ref // (BodyGenCallbackTy) is not storable (as described in the comments of // function_ref class - function_ref contains non-ownable reference // to the callable. + /// + /// \return an error, if any were triggered during execution. using StorableBodyGenCallbackTy = - std::function; + std::function; /// Callback type for loop body code generation. /// @@ -607,8 +614,10 @@ class OpenMPIRBuilder { /// terminated with an unconditional branch to the loop /// latch. /// \param IndVar is the induction variable usable at the insertion point. + /// + /// \return an error, if any were triggered during execution. using LoopBodyGenCallbackTy = - function_ref; + function_ref; /// Callback type for variable privatization (think copy & default /// constructor). @@ -628,7 +637,7 @@ class OpenMPIRBuilder { /// /// \returns The new insertion point where code generation continues and /// \p ReplVal the replacement value. - using PrivatizeCallbackTy = function_ref; @@ -658,9 +667,10 @@ class OpenMPIRBuilder { /// \param ThreadID Optional parameter to pass in any existing ThreadID value. /// /// \returns The insertion point after the barrier. - InsertPointTy createBarrier(const LocationDescription &Loc, - omp::Directive Kind, bool ForceSimpleCall = false, - bool CheckCancelFlag = true); + InsertPointOrErrorTy createBarrier(const LocationDescription &Loc, + omp::Directive Kind, + bool ForceSimpleCall = false, + bool CheckCancelFlag = true); /// Generator for '#omp cancel' /// @@ -669,8 +679,9 @@ class OpenMPIRBuilder { /// \param CanceledDirective The kind of directive that is cancled. /// /// \returns The insertion point after the barrier. - InsertPointTy createCancel(const LocationDescription &Loc, Value *IfCondition, - omp::Directive CanceledDirective); + InsertPointOrErrorTy createCancel(const LocationDescription &Loc, + Value *IfCondition, + omp::Directive CanceledDirective); /// Generator for '#omp parallel' /// @@ -685,7 +696,7 @@ class OpenMPIRBuilder { /// \param IsCancellable Flag to indicate a cancellable parallel region. /// /// \returns The insertion position *after* the parallel. - IRBuilder<>::InsertPoint + InsertPointOrErrorTy createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, @@ -711,10 +722,10 @@ class OpenMPIRBuilder { /// /// \returns An object representing the created control flow structure which /// can be used for loop-associated directives. - CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc, - LoopBodyGenCallbackTy BodyGenCB, - Value *TripCount, - const Twine &Name = "loop"); + Expected + createCanonicalLoop(const LocationDescription &Loc, + LoopBodyGenCallbackTy BodyGenCB, Value *TripCount, + const Twine &Name = "loop"); /// Generator for the control flow structure of an OpenMP canonical loop. /// @@ -764,12 +775,10 @@ class OpenMPIRBuilder { /// /// \returns An object representing the created control flow structure which /// can be used for loop-associated directives. - CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc, - LoopBodyGenCallbackTy BodyGenCB, - Value *Start, Value *Stop, Value *Step, - bool IsSigned, bool InclusiveStop, - InsertPointTy ComputeIP = {}, - const Twine &Name = "loop"); + Expected createCanonicalLoop( + const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, + Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop, + InsertPointTy ComputeIP = {}, const Twine &Name = "loop"); /// Collapse a loop nest into a single loop. /// @@ -996,9 +1005,10 @@ class OpenMPIRBuilder { /// the loop. /// /// \returns Point where to insert code after the workshare construct. - InsertPointTy applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, - InsertPointTy AllocaIP, - bool NeedsBarrier); + InsertPointOrErrorTy applyStaticWorkshareLoop(DebugLoc DL, + CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + bool NeedsBarrier); /// Modifies the canonical loop a statically-scheduled workshare loop with a /// user-specified chunk size. @@ -1013,11 +1023,11 @@ class OpenMPIRBuilder { /// \param ChunkSize The user-specified chunk size. /// /// \returns Point where to insert code after the workshare construct. - InsertPointTy applyStaticChunkedWorkshareLoop(DebugLoc DL, - CanonicalLoopInfo *CLI, - InsertPointTy AllocaIP, - bool NeedsBarrier, - Value *ChunkSize); + InsertPointOrErrorTy applyStaticChunkedWorkshareLoop(DebugLoc DL, + CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + bool NeedsBarrier, + Value *ChunkSize); /// Modifies the canonical loop to be a dynamically-scheduled workshare loop. /// @@ -1039,11 +1049,12 @@ class OpenMPIRBuilder { /// scheduling. If \p nullptr, defaults to 1. /// /// \returns Point where to insert code after the workshare construct. - InsertPointTy applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, - InsertPointTy AllocaIP, - omp::OMPScheduleType SchedType, - bool NeedsBarrier, - Value *Chunk = nullptr); + InsertPointOrErrorTy applyDynamicWorkshareLoop(DebugLoc DL, + CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + omp::OMPScheduleType SchedType, + bool NeedsBarrier, + Value *Chunk = nullptr); /// Create alternative version of the loop to support if clause /// @@ -1094,7 +1105,7 @@ class OpenMPIRBuilder { /// It corresponds to type of loop workshare OpenMP pragma. /// /// \returns Point where to insert code after the workshare construct. - InsertPointTy applyWorkshareLoop( + InsertPointOrErrorTy applyWorkshareLoop( DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier, llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default, @@ -1251,20 +1262,21 @@ class OpenMPIRBuilder { /// cannot be resumed until execution of the structured /// block that is associated with the generated task is /// completed. - InsertPointTy createTask(const LocationDescription &Loc, - InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, - bool Tied = true, Value *Final = nullptr, - Value *IfCondition = nullptr, - SmallVector Dependencies = {}); + InsertPointOrErrorTy createTask(const LocationDescription &Loc, + InsertPointTy AllocaIP, + BodyGenCallbackTy BodyGenCB, bool Tied = true, + Value *Final = nullptr, + Value *IfCondition = nullptr, + SmallVector Dependencies = {}); /// Generator for the taskgroup construct /// /// \param Loc The location where the taskgroup construct was encountered. /// \param AllocaIP The insertion point to be used for alloca instructions. /// \param BodyGenCB Callback that will generate the region code. - InsertPointTy createTaskgroup(const LocationDescription &Loc, - InsertPointTy AllocaIP, - BodyGenCallbackTy BodyGenCB); + InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc, + InsertPointTy AllocaIP, + BodyGenCallbackTy BodyGenCB); using FileIdentifierInfoCallbackTy = std::function()>; @@ -1302,15 +1314,15 @@ class OpenMPIRBuilder { /// \param CodeGenIP InsertPoint for CodeGen. /// \param LHS Pass in the LHS Value to be used for CodeGen. /// \param RHS Pass in the RHS Value to be used for CodeGen. - using ReductionGenCBTy = std::function; /// Functions used to generate atomic reductions. Such functions take two /// Values representing pointers to LHS and RHS of the reduction, as well as /// the element type of these pointers. They are expected to atomically /// update the LHS to the reduced value. - using ReductionGenAtomicCBTy = - std::function; + using ReductionGenAtomicCBTy = std::function; /// Enum class for reduction evaluation types scalar, complex and aggregate. enum class EvalKind { Scalar, Complex, Aggregate }; @@ -1510,9 +1522,10 @@ class OpenMPIRBuilder { /// need to be copied to the new function. /// /// \return The InterWarpCopy function. - Function *emitInterWarpCopyFunction(const LocationDescription &Loc, - ArrayRef ReductionInfos, - AttributeList FuncAttrs); + Expected + emitInterWarpCopyFunction(const LocationDescription &Loc, + ArrayRef ReductionInfos, + AttributeList FuncAttrs); /// This function emits a helper that copies all the reduction variables from /// the team into the provided global buffer for the reduction variables. @@ -1604,7 +1617,7 @@ class OpenMPIRBuilder { /// need to be copied to the new function. /// /// \return The reduction function. - Function *createReductionFunction( + Expected createReductionFunction( StringRef ReducerName, ArrayRef ReductionInfos, ReductionGenCBKind ReductionGenCBKind = ReductionGenCBKind::MLIR, AttributeList FuncAttrs = {}); @@ -1871,7 +1884,7 @@ class OpenMPIRBuilder { /// \param ReductionBufNum Optional OpenMPCUDAReductionBufNumValue to be /// used for teams reduction. /// \param SrcLocInfo Source location information global. - InsertPointTy createReductionsGPU( + InsertPointOrErrorTy createReductionsGPU( const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, ArrayRef ReductionInfos, bool IsNoWait = false, bool IsTeamsReduction = false, @@ -1943,10 +1956,11 @@ class OpenMPIRBuilder { /// \param IsNoWait A flag set if the reduction is marked as nowait. /// \param IsByRef A flag set if the reduction is using reference /// or direct value. - InsertPointTy createReductions(const LocationDescription &Loc, - InsertPointTy AllocaIP, - ArrayRef ReductionInfos, - ArrayRef IsByRef, bool IsNoWait = false); + InsertPointOrErrorTy createReductions(const LocationDescription &Loc, + InsertPointTy AllocaIP, + ArrayRef ReductionInfos, + ArrayRef IsByRef, + bool IsNoWait = false); ///} @@ -2002,9 +2016,11 @@ class OpenMPIRBuilder { /// \param CancelFlag Flag indicating if the cancellation is performed. /// \param CanceledDirective The kind of directive that is cancled. /// \param ExitCB Extra code to be generated in the exit block. - void emitCancelationCheckImpl(Value *CancelFlag, - omp::Directive CanceledDirective, - FinalizeCallbackTy ExitCB = {}); + /// + /// \return an error, if any were triggered during execution. + Error emitCancelationCheckImpl(Value *CancelFlag, + omp::Directive CanceledDirective, + FinalizeCallbackTy ExitCB = {}); /// Generate a target region entry call. /// @@ -2135,8 +2151,10 @@ class OpenMPIRBuilder { /// } else { /// ElseGen(); /// } - void emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, - BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {}); + /// + /// \return an error, if any were triggered during execution. + Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, + BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {}); /// Create the global variable holding the offload mappings information. GlobalVariable *createOffloadMaptypes(SmallVectorImpl &Mappings, @@ -2340,7 +2358,8 @@ class OpenMPIRBuilder { /// is executed when the kernel launch fails. It takes an insertion point as /// parameter where the code should be emitted. It returns an insertion point /// that points right after after the emitted code. - using EmitFallbackCallbackTy = function_ref; + using EmitFallbackCallbackTy = + function_ref; /// Generate a target region entry call and host fallback call. /// @@ -2352,7 +2371,7 @@ class OpenMPIRBuilder { /// \param DeviceID Identifier for the device via the 'device' clause. /// \param RTLoc Source location identifier /// \param AllocaIP The insertion point to be used for alloca instructions. - InsertPointTy + InsertPointOrErrorTy emitKernelLaunch(const LocationDescription &Loc, Value *OutlinedFnID, EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc, @@ -2366,9 +2385,11 @@ class OpenMPIRBuilder { /// \param RTLoc Source location identifier /// \Param TargetTaskAllocaIP Insertion point for the alloca block of the /// generated task. + /// + /// \return an error, if any were triggered during execution. using TargetTaskBodyCallbackTy = - function_ref; + function_ref; /// Generate a target-task for the target construct /// @@ -2380,7 +2401,7 @@ class OpenMPIRBuilder { /// dependencies as specified by the 'depend' clause. /// \param HasNoWait True if the target construct had 'nowait' on it, false /// otherwise - InsertPointTy emitTargetTask( + InsertPointOrErrorTy emitTargetTask( TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc, OpenMPIRBuilder::InsertPointTy AllocaIP, const SmallVector &Dependencies, @@ -2478,11 +2499,11 @@ class OpenMPIRBuilder { /// \param CPFuncs copy functions to use for each copyprivate variable. /// /// \returns The insertion position *after* the single call. - InsertPointTy createSingle(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB, bool IsNowait, - ArrayRef CPVars = {}, - ArrayRef CPFuncs = {}); + InsertPointOrErrorTy createSingle(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB, bool IsNowait, + ArrayRef CPVars = {}, + ArrayRef CPFuncs = {}); /// Generator for '#omp master' /// @@ -2491,9 +2512,9 @@ class OpenMPIRBuilder { /// \param FiniCB Callback to finalize variable copies. /// /// \returns The insertion position *after* the master. - InsertPointTy createMaster(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB); + InsertPointOrErrorTy createMaster(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB); /// Generator for '#omp masked' /// @@ -2502,9 +2523,9 @@ class OpenMPIRBuilder { /// \param FiniCB Callback to finialize variable copies. /// /// \returns The insertion position *after* the masked. - InsertPointTy createMasked(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB, Value *Filter); + InsertPointOrErrorTy createMasked(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB, Value *Filter); /// Generator for '#omp critical' /// @@ -2515,10 +2536,10 @@ class OpenMPIRBuilder { /// \param HintInst Hint Instruction for hint clause associated with critical /// /// \returns The insertion position *after* the critical. - InsertPointTy createCritical(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB, - StringRef CriticalName, Value *HintInst); + InsertPointOrErrorTy createCritical(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB, + StringRef CriticalName, Value *HintInst); /// Generator for '#omp ordered depend (source | sink)' /// @@ -2544,10 +2565,10 @@ class OpenMPIRBuilder { /// otherwise, with simd clause; /// /// \returns The insertion position *after* the ordered. - InsertPointTy createOrderedThreadsSimd(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB, - bool IsThreads); + InsertPointOrErrorTy createOrderedThreadsSimd(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB, + bool IsThreads); /// Generator for '#omp sections' /// @@ -2560,12 +2581,11 @@ class OpenMPIRBuilder { /// \param IsNowait If true, barrier - to ensure all sections are executed /// before moving forward will not be generated. /// \returns The insertion position *after* the sections. - InsertPointTy createSections(const LocationDescription &Loc, - InsertPointTy AllocaIP, - ArrayRef SectionCBs, - PrivatizeCallbackTy PrivCB, - FinalizeCallbackTy FiniCB, bool IsCancellable, - bool IsNowait); + InsertPointOrErrorTy + createSections(const LocationDescription &Loc, InsertPointTy AllocaIP, + ArrayRef SectionCBs, + PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, + bool IsCancellable, bool IsNowait); /// Generator for '#omp section' /// @@ -2573,9 +2593,9 @@ class OpenMPIRBuilder { /// \param BodyGenCB Callback that will generate the region body code. /// \param FiniCB Callback to finalize variable copies. /// \returns The insertion position *after* the section. - InsertPointTy createSection(const LocationDescription &Loc, - BodyGenCallbackTy BodyGenCB, - FinalizeCallbackTy FiniCB); + InsertPointOrErrorTy createSection(const LocationDescription &Loc, + BodyGenCallbackTy BodyGenCB, + FinalizeCallbackTy FiniCB); /// Generator for `#omp teams` /// @@ -2589,7 +2609,7 @@ class OpenMPIRBuilder { /// contention group created by each team. /// \param IfExpr is the integer argument value of the if condition on the /// teams clause. - InsertPointTy + InsertPointOrErrorTy createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower = nullptr, Value *NumTeamsUpper = nullptr, Value *ThreadLimit = nullptr, Value *IfExpr = nullptr); @@ -2764,7 +2784,8 @@ class OpenMPIRBuilder { public: /// Functions used to generate a function with the given name. - using FunctionGenCallback = std::function; + using FunctionGenCallback = + std::function(StringRef FunctionName)>; /// Create a unique name for the entry function using the source location /// information of the current target region. The name will be something like: @@ -2797,10 +2818,10 @@ class OpenMPIRBuilder { /// \param GenerateFunctionCallback The callback function to generate the code /// \param OutlinedFunction Pointer to the outlined function /// \param EntryFnIDName Name of the ID o be created - void emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, - FunctionGenCallback &GenerateFunctionCallback, - bool IsOffloadEntry, Function *&OutlinedFn, - Constant *&OutlinedFnID); + Error emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, + FunctionGenCallback &GenerateFunctionCallback, + bool IsOffloadEntry, Function *&OutlinedFn, + Constant *&OutlinedFnID); /// Registers the given function and sets up the attribtues of the function /// Returns the FunctionID. @@ -2851,22 +2872,22 @@ class OpenMPIRBuilder { /// use_device_ptr and use_device_addr. /// \param CustomMapperCB Optional callback to generate code related to /// custom mappers. - OpenMPIRBuilder::InsertPointTy createTargetData( + InsertPointOrErrorTy createTargetData( const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond, TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB, omp::RuntimeFunction *MapperFunc = nullptr, - function_ref + function_ref BodyGenCB = nullptr, function_ref DeviceAddrCB = nullptr, function_ref CustomMapperCB = nullptr, Value *SrcLocInfo = nullptr); - using TargetBodyGenCallbackTy = function_ref; - using TargetGenArgAccessorsCallbackTy = function_ref; @@ -2887,7 +2908,7 @@ class OpenMPIRBuilder { /// \param Dependencies A vector of DependData objects that carry // dependency information as passed in the depend clause // \param HasNowait Whether the target construct has a `nowait` clause or not. - InsertPointTy createTarget( + InsertPointOrErrorTy createTarget( const LocationDescription &Loc, bool IsOffloadEntry, OpenMPIRBuilder::InsertPointTy AllocaIP, OpenMPIRBuilder::InsertPointTy CodeGenIP, @@ -2969,10 +2990,10 @@ class OpenMPIRBuilder { /// should be called. /// /// \return The insertion position in exit block - InsertPointTy emitCommonDirectiveExit(omp::Directive OMPD, - InsertPointTy FinIP, - Instruction *ExitCall, - bool HasFinalize = true); + InsertPointOrErrorTy emitCommonDirectiveExit(omp::Directive OMPD, + InsertPointTy FinIP, + Instruction *ExitCall, + bool HasFinalize = true); /// Common Interface to generate OMP inlined regions /// @@ -2990,8 +3011,7 @@ class OpenMPIRBuilder { /// \param IsCancellable if HasFinalize is set to true, indicate if the /// the directive should be cancellable. /// \return The insertion point after the region - - InsertPointTy + InsertPointOrErrorTy EmitOMPInlinedRegion(omp::Directive OMPD, Instruction *EntryCall, Instruction *ExitCall, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional = false, @@ -3027,7 +3047,7 @@ class OpenMPIRBuilder { /// /// \returns Value to update X to. using AtomicUpdateCallbackTy = - const function_ref &IRB)>; + const function_ref(Value *XOld, IRBuilder<> &IRB)>; private: enum AtomicKind { Read, Write, Update, Capture, Compare }; @@ -3066,7 +3086,7 @@ class OpenMPIRBuilder { /// /// \returns A pair of the old value of X before the update, and the value /// used for the update. - std::pair + Expected> emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, @@ -3143,12 +3163,11 @@ class OpenMPIRBuilder { /// (e.g. true for X = X BinOp Expr) /// /// \return Insertion point after generated atomic update IR. - InsertPointTy createAtomicUpdate(const LocationDescription &Loc, - InsertPointTy AllocaIP, AtomicOpValue &X, - Value *Expr, AtomicOrdering AO, - AtomicRMWInst::BinOp RMWOp, - AtomicUpdateCallbackTy &UpdateOp, - bool IsXBinopExpr); + InsertPointOrErrorTy + createAtomicUpdate(const LocationDescription &Loc, InsertPointTy AllocaIP, + AtomicOpValue &X, Value *Expr, AtomicOrdering AO, + AtomicRMWInst::BinOp RMWOp, + AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr); /// Emit atomic update for constructs: --- Only Scalar data types /// V = X; X = X BinOp Expr , @@ -3179,7 +3198,7 @@ class OpenMPIRBuilder { /// 'v', not an updated one. /// /// \return Insertion point after generated atomic capture IR. - InsertPointTy + InsertPointOrErrorTy createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, AtomicOpValue &V, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index be93f9f2e1fdc8..d2e4dc1c85dfd2 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -945,7 +945,7 @@ Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) { "omp_global_thread_num"); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind, bool ForceSimpleCall, bool CheckCancelFlag) { if (!updateToLocation(Loc)) @@ -992,12 +992,13 @@ OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind, Args); if (UseCancelBarrier && CheckCancelFlag) - emitCancelationCheckImpl(Result, OMPD_parallel); + if (Error Err = emitCancelationCheckImpl(Result, OMPD_parallel)) + return Err; return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCancel(const LocationDescription &Loc, Value *IfCondition, omp::Directive CanceledDirective) { @@ -1029,18 +1030,22 @@ OpenMPIRBuilder::createCancel(const LocationDescription &Loc, Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind}; Value *Result = Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args); - auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) { + auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) -> Error { if (CanceledDirective == OMPD_parallel) { IRBuilder<>::InsertPointGuard IPG(Builder); Builder.restoreIP(IP); - createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), - omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false, - /* CheckCancelFlag */ false); + return createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), + omp::Directive::OMPD_unknown, + /* ForceSimpleCall */ false, + /* CheckCancelFlag */ false) + .takeError(); } + return Error::success(); }; // The actual cancel logic is shared with others, e.g., cancel_barriers. - emitCancelationCheckImpl(Result, CanceledDirective, ExitCB); + if (Error Err = emitCancelationCheckImpl(Result, CanceledDirective, ExitCB)) + return Err; // Update the insertion point and remove the terminator we introduced. Builder.SetInsertPoint(UI->getParent()); @@ -1079,7 +1084,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel( return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch( const LocationDescription &Loc, Value *OutlinedFnID, EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) { @@ -1134,15 +1139,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch( auto CurFn = Builder.GetInsertBlock()->getParent(); emitBlock(OffloadFailedBlock, CurFn); - Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP())); + InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP()); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); emitBranch(OffloadContBlock); emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true); return Builder.saveIP(); } -void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag, - omp::Directive CanceledDirective, - FinalizeCallbackTy ExitCB) { +Error OpenMPIRBuilder::emitCancelationCheckImpl( + Value *CancelFlag, omp::Directive CanceledDirective, + FinalizeCallbackTy ExitCB) { assert(isLastFinalizationInfoCancellable(CanceledDirective) && "Unexpected cancellation!"); @@ -1171,12 +1179,15 @@ void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag, // post finalization block that is known to the FiniCB callback. Builder.SetInsertPoint(CancellationBlock); if (ExitCB) - ExitCB(Builder.saveIP()); + if (Error Err = ExitCB(Builder.saveIP())) + return Err; auto &FI = FinalizationStack.back(); - FI.FiniCB(Builder.saveIP()); + if (Error Err = FI.FiniCB(Builder.saveIP())) + return Err; // The continuation block is where code generation continues. Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); + return Error::success(); } // Callback used to create OpenMP runtime calls to support @@ -1355,7 +1366,7 @@ hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, } } -IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel( const LocationDescription &Loc, InsertPointTy OuterAllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads, @@ -1496,7 +1507,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( // Let the caller create the body. assert(BodyGenCB && "Expected body generation callback!"); InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin()); - BodyGenCB(InnerAllocaIP, CodeGenIP); + if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP)) + return Err; LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n"); @@ -1565,10 +1577,10 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( FunctionCallee TIDRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num); - auto PrivHelper = [&](Value &V) { + auto PrivHelper = [&](Value &V) -> Error { if (&V == TIDAddr || &V == ZeroAddr) { OI.ExcludeArgsFromAggregate.push_back(&V); - return; + return Error::success(); } SetVector Uses; @@ -1608,8 +1620,11 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) { ReplacementValue = PrivTID; } else { - Builder.restoreIP( - PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue)); + InsertPointOrErrorTy AfterIP = + PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); InnerAllocaIP = { InnerAllocaIP.getBlock(), InnerAllocaIP.getBlock()->getTerminator()->getIterator()}; @@ -1617,11 +1632,13 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( assert(ReplacementValue && "Expected copy/create callback to set replacement value!"); if (ReplacementValue == &V) - return; + return Error::success(); } for (Use *UPtr : Uses) UPtr->set(ReplacementValue); + + return Error::success(); }; // Reset the inner alloca insertion as it will be used for loading the values @@ -1640,7 +1657,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( for (Value *Input : Inputs) { LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n"); - PrivHelper(*Input); + if (Error Err = PrivHelper(*Input)) + return Err; } LLVM_DEBUG({ for (Value *Output : Outputs) @@ -1666,7 +1684,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); - FiniCB(PreFiniIP); + if (Error Err = FiniCB(PreFiniIP)) + return Err; // Register the outlined info. addOutlineInfo(std::move(OI)); @@ -1797,7 +1816,7 @@ static Value *emitTaskDependencies( return DepArray; } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, bool Tied, Value *Final, Value *IfCondition, @@ -1833,7 +1852,8 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc, InsertPointTy TaskAllocaIP = InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin()); InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin()); - BodyGenCB(TaskAllocaIP, TaskBodyIP); + if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP)) + return Err; OutlineInfo OI; OI.EntryBB = TaskAllocaBB; @@ -2048,7 +2068,7 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB) { @@ -2066,7 +2086,8 @@ OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc, Builder.CreateCall(TaskgroupFn, {Ident, ThreadID}); BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit"); - BodyGenCB(AllocaIP, Builder.saveIP()); + if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP())) + return Err; Builder.SetInsertPoint(TaskgroupExitBB); // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup @@ -2077,7 +2098,7 @@ OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections( const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef SectionCBs, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) { @@ -2124,7 +2145,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections( // ... // section_loop.after: // ; - auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) { + auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error { Builder.restoreIP(CodeGenIP); BasicBlock *Continue = splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after"); @@ -2138,12 +2159,14 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections( SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB); Builder.SetInsertPoint(CaseBB); BranchInst *CaseEndBr = Builder.CreateBr(Continue); - SectionCB(InsertPointTy(), - {CaseEndBr->getParent(), CaseEndBr->getIterator()}); + if (Error Err = SectionCB(InsertPointTy(), {CaseEndBr->getParent(), + CaseEndBr->getIterator()})) + return Err; CaseNumber++; } // remove the existing terminator from body BB since there can be no // terminators after switch/case + return Error::success(); }; // Loop body ends here // LowerBound, UpperBound, and STride for createCanonicalLoop @@ -2151,10 +2174,16 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections( Value *LB = ConstantInt::get(I32Ty, 0); Value *UB = ConstantInt::get(I32Ty, SectionCBs.size()); Value *ST = ConstantInt::get(I32Ty, 1); - llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop( + Expected LoopInfo = createCanonicalLoop( Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop"); - InsertPointTy AfterIP = - applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait); + if (!LoopInfo) + return LoopInfo.takeError(); + + InsertPointOrErrorTy WsloopIP = + applyStaticWorkshareLoop(Loc.DL, *LoopInfo, AllocaIP, !IsNowait); + if (!WsloopIP) + return WsloopIP.takeError(); + InsertPointTy AfterIP = *WsloopIP; // Apply the finalization callback in LoopAfterBB auto FiniInfo = FinalizationStack.pop_back_val(); @@ -2164,14 +2193,15 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections( Builder.restoreIP(AfterIP); BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini"); - CB(Builder.saveIP()); + if (Error Err = CB(Builder.saveIP())) + return Err; AfterIP = {FiniBB, FiniBB->begin()}; } return AfterIP; } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSection(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB) { @@ -2502,7 +2532,7 @@ void OpenMPIRBuilder::emitReductionListCopy( } } -Function *OpenMPIRBuilder::emitInterWarpCopyFunction( +Expected OpenMPIRBuilder::emitInterWarpCopyFunction( const LocationDescription &Loc, ArrayRef ReductionInfos, AttributeList FuncAttrs) { InsertPointTy SavedIP = Builder.saveIP(); @@ -2621,10 +2651,13 @@ Function *OpenMPIRBuilder::emitInterWarpCopyFunction( } // kmpc_barrier. - createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), - omp::Directive::OMPD_unknown, - /* ForceSimpleCall */ false, - /* CheckCancelFlag */ true); + InsertPointOrErrorTy BarrierIP1 = + createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), + omp::Directive::OMPD_unknown, + /* ForceSimpleCall */ false, + /* CheckCancelFlag */ true); + if (!BarrierIP1) + return BarrierIP1.takeError(); BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then"); BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else"); BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont"); @@ -2666,10 +2699,13 @@ Function *OpenMPIRBuilder::emitInterWarpCopyFunction( // endif emitBlock(MergeBB, Builder.GetInsertBlock()->getParent()); - createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), - omp::Directive::OMPD_unknown, - /* ForceSimpleCall */ false, - /* CheckCancelFlag */ true); + InsertPointOrErrorTy BarrierIP2 = + createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), + omp::Directive::OMPD_unknown, + /* ForceSimpleCall */ false, + /* CheckCancelFlag */ true); + if (!BarrierIP2) + return BarrierIP2.takeError(); // Warp 0 copies reduce element from transfer medium BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "then"); @@ -3286,7 +3322,7 @@ std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const { return (Name + Suffix).str(); } -Function *OpenMPIRBuilder::createReductionFunction( +Expected OpenMPIRBuilder::createReductionFunction( StringRef ReducerName, ArrayRef ReductionInfos, ReductionGenCBKind ReductionGenCBKind, AttributeList FuncAttrs) { auto *FuncTy = FunctionType::get(Builder.getVoidTy(), @@ -3352,7 +3388,10 @@ Function *OpenMPIRBuilder::createReductionFunction( Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr); Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr); Value *Reduced; - RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced); + InsertPointOrErrorTy AfterIP = + RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced); + if (!AfterIP) + return AfterIP.takeError(); if (!Builder.GetInsertBlock()) return ReductionFunc; Builder.CreateStore(Reduced, LHSPtr); @@ -3405,7 +3444,7 @@ checkReductionInfos(ArrayRef ReductionInfos, } } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU( const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, ArrayRef ReductionInfos, bool IsNoWait, bool IsTeamsReduction, bool HasDistribute, @@ -3435,11 +3474,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU( AttrBldr.removeAttribute(Attribute::OptimizeNone); FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr); - Function *ReductionFunc = nullptr; CodeGenIP = Builder.saveIP(); - ReductionFunc = + Expected ReductionResult = createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, ReductionGenCBKind, FuncAttrs); + if (!ReductionResult) + return ReductionResult.takeError(); + Function *ReductionFunc = *ReductionResult; Builder.restoreIP(CodeGenIP); // Set the grid value in the config needed for lowering later on @@ -3480,7 +3521,11 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU( CodeGenIP = Builder.saveIP(); Function *SarFunc = emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs); - Function *WcFunc = emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs); + Expected CopyResult = + emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs); + if (!CopyResult) + return CopyResult.takeError(); + Function *WcFunc = *CopyResult; Builder.restoreIP(CodeGenIP); Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy); @@ -3595,7 +3640,7 @@ static Function *getFreshReductionFunc(Module &M) { ".omp.reduction.func", &M); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef ReductionInfos, @@ -3688,7 +3733,7 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc, Type *ValueType = RI.ElementType; // We have one less load for by-ref case because that load is now inside of // the reduction region - Value *RedValue = nullptr; + Value *RedValue = RI.Variable; if (!IsByRef[En.index()]) { RedValue = Builder.CreateLoad(ValueType, RI.Variable, "red.value." + Twine(En.index())); @@ -3697,13 +3742,12 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc, Builder.CreateLoad(ValueType, RI.PrivateVariable, "red.private.value." + Twine(En.index())); Value *Reduced; - if (IsByRef[En.index()]) { - Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable, - PrivateRedValue, Reduced)); - } else { - Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue, - PrivateRedValue, Reduced)); - } + InsertPointOrErrorTy AfterIP = + RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); + if (!Builder.GetInsertBlock()) return InsertPointTy(); // for by-ref case, the load is inside of the reduction region @@ -3722,8 +3766,11 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc, Builder.SetInsertPoint(AtomicRedBlock); if (CanGenerateAtomic && llvm::none_of(IsByRef, [](bool P) { return P; })) { for (const ReductionInfo &RI : ReductionInfos) { - Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType, - RI.Variable, RI.PrivateVariable)); + InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen( + Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); if (!Builder.GetInsertBlock()) return InsertPointTy(); } @@ -3755,7 +3802,11 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc, Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType()); Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr); Value *Reduced; - Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced)); + InsertPointOrErrorTy AfterIP = + RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); if (!Builder.GetInsertBlock()) return InsertPointTy(); // store is inside of the reduction region when using by-ref @@ -3768,11 +3819,10 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createMaster(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB) { - if (!updateToLocation(Loc)) return Loc.IP; @@ -3793,7 +3843,7 @@ OpenMPIRBuilder::createMaster(const LocationDescription &Loc, /*Conditional*/ true, /*hasFinalize*/ true); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createMasked(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, Value *Filter) { @@ -3884,7 +3934,7 @@ CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton( return CL; } -CanonicalLoopInfo * +Expected OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, Value *TripCount, const Twine &Name) { @@ -3906,7 +3956,8 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc, // Emit the body content. We do it after connecting the loop to the CFG to // avoid that the callback encounters degenerate BBs. - BodyGenCB(CL->getBodyIP(), CL->getIndVar()); + if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar())) + return Err; #ifndef NDEBUG CL->assertOK(); @@ -3914,7 +3965,7 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc, return CL; } -CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop( +Expected OpenMPIRBuilder::createCanonicalLoop( const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop, InsertPointTy ComputeIP, const Twine &Name) { @@ -3979,7 +4030,7 @@ CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop( Builder.restoreIP(CodeGenIP); Value *Span = Builder.CreateMul(IV, Step); Value *IndVar = Builder.CreateAdd(Span, Start); - BodyGenCB(Builder.saveIP(), IndVar); + return BodyGenCB(Builder.saveIP(), IndVar); }; LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP(); return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name); @@ -4001,7 +4052,7 @@ static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M, llvm_unreachable("unknown OpenMP loop iterator bitwidth"); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier) { @@ -4078,10 +4129,14 @@ OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum}); // Add the barrier if requested. - if (NeedsBarrier) - createBarrier(LocationDescription(Builder.saveIP(), DL), - omp::Directive::OMPD_for, /* ForceSimpleCall */ false, - /* CheckCancelFlag */ false); + if (NeedsBarrier) { + InsertPointOrErrorTy BarrierIP = + createBarrier(LocationDescription(Builder.saveIP(), DL), + omp::Directive::OMPD_for, /* ForceSimpleCall */ false, + /* CheckCancelFlag */ false); + if (!BarrierIP) + return BarrierIP.takeError(); + } InsertPointTy AfterIP = CLI->getAfterIP(); CLI->invalidate(); @@ -4089,9 +4144,12 @@ OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, return AfterIP; } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop( - DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, - bool NeedsBarrier, Value *ChunkSize) { +OpenMPIRBuilder::InsertPointOrErrorTy +OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(DebugLoc DL, + CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + bool NeedsBarrier, + Value *ChunkSize) { assert(CLI->isValid() && "Requires a valid canonical loop"); assert(ChunkSize && "Chunk size is required"); @@ -4167,12 +4225,23 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop( // Create outer "dispatch" loop for enumerating the chunks. BasicBlock *DispatchEnter = splitBB(Builder, true); Value *DispatchCounter; - CanonicalLoopInfo *DispatchCLI = createCanonicalLoop( + Expected LoopResult = createCanonicalLoop( {Builder.saveIP(), DL}, - [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; }, + [&](InsertPointTy BodyIP, Value *Counter) { + DispatchCounter = Counter; + return Error::success(); + }, FirstChunkStart, CastedTripCount, NextChunkStride, /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{}, "dispatch"); + if (!LoopResult) { + // It is safe to assume this didn't return an error because the callback + // passed into createCanonicalLoop is the only possible error source, and it + // always returns success. Need to still cast the result into bool to avoid + // runtime errors. + llvm_unreachable("unexpected error creating canonical loop"); + } + CanonicalLoopInfo *DispatchCLI = *LoopResult; // Remember the BasicBlocks of the dispatch loop we need, then invalidate to // not have to preserve the canonical invariant. @@ -4219,9 +4288,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop( Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum}); // Add the barrier if requested. - if (NeedsBarrier) - createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for, - /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false); + if (NeedsBarrier) { + InsertPointOrErrorTy AfterIP = + createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for, + /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false); + if (!AfterIP) + return AfterIP.takeError(); + } #ifndef NDEBUG // Even though we currently do not support applying additional methods to it, @@ -4229,7 +4302,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop( CLI->assertOK(); #endif - return {DispatchAfter, DispatchAfter->getFirstInsertionPt()}; + return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt()); } // Returns an LLVM function to call for executing an OpenMP static worksharing @@ -4462,7 +4535,7 @@ OpenMPIRBuilder::applyWorkshareLoopTarget(DebugLoc DL, CanonicalLoopInfo *CLI, return CLI->getAfterIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop( DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize, bool HasSimdModifier, bool HasMonotonicModifier, @@ -4563,9 +4636,11 @@ getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) { llvm_unreachable("unknown OpenMP loop iterator bitwidth"); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop( - DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, - OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) { +OpenMPIRBuilder::InsertPointOrErrorTy +OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, + InsertPointTy AllocaIP, + OMPScheduleType SchedType, + bool NeedsBarrier, Value *Chunk) { assert(CLI->isValid() && "Requires a valid canonical loop"); assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) && "Require dedicated allocate IP"); @@ -4681,9 +4756,12 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop( // Add the barrier if requested. if (NeedsBarrier) { Builder.SetInsertPoint(&Exit->back()); - createBarrier(LocationDescription(Builder.saveIP(), DL), - omp::Directive::OMPD_for, /* ForceSimpleCall */ false, - /* CheckCancelFlag */ false); + InsertPointOrErrorTy BarrierIP = + createBarrier(LocationDescription(Builder.saveIP(), DL), + omp::Directive::OMPD_for, /* ForceSimpleCall */ false, + /* CheckCancelFlag */ false); + if (!BarrierIP) + return BarrierIP.takeError(); } CLI->invalidate(); @@ -5542,7 +5620,7 @@ OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle( const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef CPVars, ArrayRef CPFuncs) { @@ -5571,14 +5649,17 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle( Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single); Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); - auto FiniCBWrapper = [&](InsertPointTy IP) { - FiniCB(IP); + auto FiniCBWrapper = [&](InsertPointTy IP) -> Error { + if (Error Err = FiniCB(IP)) + return Err; // The thread that executes the single region must set `DidIt` to 1. // This is used by __kmpc_copyprivate, to know if the caller is the // single thread or not. if (DidIt) Builder.CreateStore(Builder.getInt32(1), DidIt); + + return Error::success(); }; // generates the following: @@ -5589,9 +5670,12 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle( // __kmpc_copyprivate // __kmpc_barrier - EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper, - /*Conditional*/ true, - /*hasFinalize*/ true); + InsertPointOrErrorTy AfterIP = + EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper, + /*Conditional*/ true, + /*hasFinalize*/ true); + if (!AfterIP) + return AfterIP.takeError(); if (DidIt) { for (size_t I = 0, E = CPVars.size(); I < E; ++I) @@ -5600,14 +5684,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle( /*BufSize=*/ConstantInt::get(Int64, 0), CPVars[I], CPFuncs[I], DidIt); // NOTE __kmpc_copyprivate already inserts a barrier - } else if (!IsNowait) - createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), - omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false, - /* CheckCancelFlag */ false); + } else if (!IsNowait) { + InsertPointOrErrorTy AfterIP = + createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), + omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false, + /* CheckCancelFlag */ false); + if (!AfterIP) + return AfterIP.takeError(); + } return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical( const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) { @@ -5688,7 +5776,7 @@ OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd( const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsThreads) { if (!updateToLocation(Loc)) @@ -5717,7 +5805,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd( /*Conditional*/ false, /*hasFinalize*/ true); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion( Directive OMPD, Instruction *EntryCall, Instruction *ExitCall, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional, bool HasFinalize, bool IsCancellable) { @@ -5739,15 +5827,19 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion( emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional); // generate body - BodyGenCB(/* AllocaIP */ InsertPointTy(), - /* CodeGenIP */ Builder.saveIP()); + if (Error Err = BodyGenCB(/* AllocaIP */ InsertPointTy(), + /* CodeGenIP */ Builder.saveIP())) + return Err; // emit exit call and do any needed finalization. auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt()); assert(FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && "Unexpected control flow graph state!!"); - emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize); + InsertPointOrErrorTy AfterIP = + emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize); + if (!AfterIP) + return AfterIP.takeError(); assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"); MergeBlockIntoPredecessor(FiniBB); @@ -5796,7 +5888,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry( return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt()); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit( omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall, bool HasFinalize) { @@ -5810,7 +5902,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit( FinalizationInfo Fi = FinalizationStack.pop_back_val(); assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!"); - Fi.FiniCB(FinIP); + if (Error Err = Fi.FiniCB(FinIP)) + return Err; BasicBlock *FiniBB = FinIP.getBlock(); Instruction *FiniBBTI = FiniBB->getTerminator(); @@ -6319,7 +6412,7 @@ Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn, Constant::getNullValue(Builder.getInt8Ty()), EntryFnName); } -void OpenMPIRBuilder::emitTargetRegionFunction( +Error OpenMPIRBuilder::emitTargetRegionFunction( TargetRegionEntryInfo &EntryInfo, FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry, Function *&OutlinedFn, Constant *&OutlinedFnID) { @@ -6327,15 +6420,20 @@ void OpenMPIRBuilder::emitTargetRegionFunction( SmallString<64> EntryFnName; OffloadInfoManager.getTargetRegionEntryFnName(EntryFnName, EntryInfo); - OutlinedFn = Config.isTargetDevice() || !Config.openMPOffloadMandatory() - ? GenerateFunctionCallback(EntryFnName) - : nullptr; + if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) { + Expected CBResult = GenerateFunctionCallback(EntryFnName); + if (!CBResult) + return CBResult.takeError(); + OutlinedFn = *CBResult; + } else { + OutlinedFn = nullptr; + } // If this target outline function is not an offload entry, we don't need to // register it. This may be in the case of a false if clause, or if there are // no OpenMP targets. if (!IsOffloadEntry) - return; + return Error::success(); std::string EntryFnIDName = Config.isTargetDevice() @@ -6344,6 +6442,7 @@ void OpenMPIRBuilder::emitTargetRegionFunction( OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFn, EntryFnName, EntryFnIDName); + return Error::success(); } Constant *OpenMPIRBuilder::registerTargetRegionFunction( @@ -6359,12 +6458,13 @@ Constant *OpenMPIRBuilder::registerTargetRegionFunction( return OutlinedFnID; } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData( const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond, TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB, omp::RuntimeFunction *MapperFunc, - function_ref + function_ref BodyGenCB, function_ref DeviceAddrCB, function_ref CustomMapperCB, Value *SrcLocInfo) { @@ -6374,8 +6474,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( Builder.restoreIP(CodeGenIP); // Disable TargetData CodeGen on Device pass. if (Config.IsTargetDevice.value_or(false)) { - if (BodyGenCB) - Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv)); + if (BodyGenCB) { + InsertPointOrErrorTy AfterIP = + BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); + } return Builder.saveIP(); } @@ -6384,7 +6489,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( // Generate the code for the opening of the data environment. Capture all the // arguments of the runtime call by reference because they are used in the // closing of the region. - auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + auto BeginThenGen = [&](InsertPointTy AllocaIP, + InsertPointTy CodeGenIP) -> Error { MapInfo = &GenMapInfoCB(Builder.saveIP()); emitOffloadingArrays(AllocaIP, Builder.saveIP(), *MapInfo, Info, /*IsNonContiguous=*/true, DeviceAddrCB, @@ -6413,7 +6519,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( if (IsStandAlone) { assert(MapperFunc && "MapperFunc missing for standalone target data"); - auto TaskBodyCB = [&](Value *, Value *, IRBuilderBase::InsertPoint) { + auto TaskBodyCB = [&](Value *, Value *, + IRBuilderBase::InsertPoint) -> Error { if (Info.HasNoWait) { OffloadingArgs.append({llvm::Constant::getNullValue(Int32), llvm::Constant::getNullValue(VoidPtr), @@ -6431,16 +6538,20 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true); Builder.restoreIP(Builder.saveIP()); } + return Error::success(); }; bool RequiresOuterTargetTask = Info.HasNoWait; - - if (!RequiresOuterTargetTask) - TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr, - /*TargetTaskAllocaIP=*/{}); - else - emitTargetTask(TaskBodyCB, DeviceID, SrcLocInfo, AllocaIP, - /*Dependencies=*/{}, Info.HasNoWait); + if (!RequiresOuterTargetTask) { + Error Err = TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr, + /*TargetTaskAllocaIP=*/{}); + assert(!Err && "TaskBodyCB expected to succeed"); + } else { + InsertPointOrErrorTy AfterIP = + emitTargetTask(TaskBodyCB, DeviceID, SrcLocInfo, AllocaIP, + /*Dependencies=*/{}, Info.HasNoWait); + assert(AfterIP && "TaskBodyCB expected to succeed"); + } } else { Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr( omp::OMPRTL___tgt_target_data_begin_mapper); @@ -6458,15 +6569,26 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( // If device pointer privatization is required, emit the body of the // region here. It will have to be duplicated: with and without // privatization. - Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv)); + InsertPointOrErrorTy AfterIP = + BodyGenCB(Builder.saveIP(), BodyGenTy::Priv); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); } + return Error::success(); }; // If we need device pointer privatization, we need to emit the body of the // region with no privatization in the 'else' branch of the conditional. // Otherwise, we don't have to do anything. - auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { - Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv)); + auto BeginElseGen = [&](InsertPointTy AllocaIP, + InsertPointTy CodeGenIP) -> Error { + InsertPointOrErrorTy AfterIP = + BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); + return Error::success(); }; // Generate code for the closing of the data region. @@ -6494,35 +6616,45 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData( getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper); Builder.CreateCall(EndMapperFunc, OffloadingArgs); + return Error::success(); }; // We don't have to do anything to close the region if the if clause evaluates // to false. - auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; - if (BodyGenCB) { - if (IfCond) { - emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP); - } else { - BeginThenGen(AllocaIP, Builder.saveIP()); + Error Err = [&]() -> Error { + if (BodyGenCB) { + Error Err = [&]() { + if (IfCond) + return emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP); + return BeginThenGen(AllocaIP, Builder.saveIP()); + }(); + + if (Err) + return Err; + + // If we don't require privatization of device pointers, we emit the body + // in between the runtime calls. This avoids duplicating the body code. + InsertPointOrErrorTy AfterIP = + BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); + + if (IfCond) + return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP); + return EndThenGen(AllocaIP, Builder.saveIP()); } + if (IfCond) + return emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP); + return BeginThenGen(AllocaIP, Builder.saveIP()); + }(); - // If we don't require privatization of device pointers, we emit the body in - // between the runtime calls. This avoids duplicating the body code. - Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv)); - - if (IfCond) { - emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP); - } else { - EndThenGen(AllocaIP, Builder.saveIP()); - } - } else { - if (IfCond) { - emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP); - } else { - BeginThenGen(AllocaIP, Builder.saveIP()); - } - } + if (Err) + return Err; return Builder.saveIP(); } @@ -6591,7 +6723,7 @@ FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() { return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit); } -static Function *createOutlinedFunction( +static Expected createOutlinedFunction( OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName, SmallVectorImpl &Inputs, OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc, @@ -6671,7 +6803,11 @@ static Function *createOutlinedFunction( OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Func); // Insert target deinit call in the device compilation pass. - Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + CBFunc(Builder.saveIP(), Builder.saveIP()); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); if (OMPBuilder.Config.isTargetDevice()) OMPBuilder.createTargetDeinit(Builder); @@ -6726,8 +6862,11 @@ static Function *createOutlinedFunction( Argument &Arg = std::get<1>(InArg); Value *InputCopy = nullptr; - Builder.restoreIP( - ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP()); + if (!AfterIP) + return AfterIP.takeError(); + Builder.restoreIP(*AfterIP); // In certain cases a Global may be set up for replacement, however, this // Global may be used in multiple arguments to the kernel, just segmented @@ -6847,7 +6986,8 @@ static Function *emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder, Builder.CreateRetVoid(); return ProxyFn; } -static void emitTargetOutlinedFunction( + +static Error emitTargetOutlinedFunction( OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry, TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn, Constant *&OutlinedFnID, SmallVectorImpl &Inputs, @@ -6861,11 +7001,12 @@ static void emitTargetOutlinedFunction( CBFunc, ArgAccessorFuncCB); }; - OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction, - IsOffloadEntry, OutlinedFn, OutlinedFnID); + return OMPBuilder.emitTargetRegionFunction( + EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn, + OutlinedFnID); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask( TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc, OpenMPIRBuilder::InsertPointTy AllocaIP, const SmallVector &Dependencies, @@ -6983,7 +7124,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask( Builder.restoreIP(TargetTaskBodyIP); - TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP); + if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP)) + return Err; OI.ExitBB = Builder.saveIP().getBlock(); OI.PostOutlineCB = [this, ToBeDeleted, Dependencies, HasNoWait, @@ -7161,8 +7303,8 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, // Generate a function call to the host fallback implementation of the target // region. This is called by the host when no offload entry was generated for // the target region and when the offloading call fails at runtime. - auto &&EmitTargetCallFallbackCB = - [&](OpenMPIRBuilder::InsertPointTy IP) -> OpenMPIRBuilder::InsertPointTy { + auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP) + -> OpenMPIRBuilder::InsertPointOrErrorTy { Builder.restoreIP(IP); Builder.CreateCall(OutlinedFn, Args); return Builder.saveIP(); @@ -7173,9 +7315,10 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, OpenMPIRBuilder::TargetKernelArgs KArgs; - auto TaskBodyCB = [&](Value *DeviceID, Value *RTLoc, - IRBuilderBase::InsertPoint TargetTaskAllocaIP) { - if (OutlinedFnID) { + auto TaskBodyCB = + [&](Value *DeviceID, Value *RTLoc, + IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error { + llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() { // emitKernelLaunch makes the necessary runtime call to offload the // kernel. We then outline all that code into a separate function // ('kernel_launch_function' in the pseudo code above). This function is @@ -7183,31 +7326,41 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, // '@.omp_target_task_proxy_func' in the pseudo code above) // "@.omp_target_task_proxy_func' is generated by // emitTargetTaskProxyFunction. - Builder.restoreIP(OMPBuilder.emitKernelLaunch( - Builder, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID, - RTLoc, TargetTaskAllocaIP)); - } else { - // When OutlinedFnID is set to nullptr, then it's not an offloading - // call. In this case, we execute the host implementation directly. - OMPBuilder.Builder.restoreIP( - EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP())); - } + if (OutlinedFnID) + return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID, + EmitTargetCallFallbackCB, KArgs, + DeviceID, RTLoc, TargetTaskAllocaIP); + // When OutlinedFnID is set to nullptr, then it's not an offloading call. + // In this case, we execute the host implementation directly. + return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP()); + }(); + + if (!AfterIP) + return AfterIP.takeError(); + + OMPBuilder.Builder.restoreIP(*AfterIP); + return Error::success(); }; // If we don't have an ID for the target region, it means an offload entry // wasn't created. In this case we just run the host fallback directly. if (!OutlinedFnID) { - if (RequiresOuterTargetTask) { - // Arguments that are intended to be directly forwarded to an - // emitKernelLaunch call are pased as nullptr, since OutlinedFnID=nullptr - // results in that call not being done. - Builder.restoreIP(OMPBuilder.emitTargetTask(TaskBodyCB, - /*DeviceID=*/nullptr, - /*RTLoc=*/nullptr, AllocaIP, - Dependencies, HasNoWait)); - } else { - Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP())); - } + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() { + if (RequiresOuterTargetTask) { + // Arguments that are intended to be directly forwarded to an + // emitKernelLaunch call are pased as nullptr, since + // OutlinedFnID=nullptr results in that call not being done. + return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr, + /*RTLoc=*/nullptr, AllocaIP, + Dependencies, HasNoWait); + } + return EmitTargetCallFallbackCB(Builder.saveIP()); + }(); + + // Assume no error was returned because EmitTargetCallFallbackCB doesn't + // produce any. The 'if' check enables accessing the returned value. + if (AfterIP) + Builder.restoreIP(*AfterIP); return; } @@ -7247,17 +7400,24 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, // The presence of certain clauses on the target directive require the // explicit generation of the target task. - if (RequiresOuterTargetTask) { - Builder.restoreIP(OMPBuilder.emitTargetTask( - TaskBodyCB, DeviceID, RTLoc, AllocaIP, Dependencies, HasNoWait)); - } else { - Builder.restoreIP(OMPBuilder.emitKernelLaunch( - Builder, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID, RTLoc, - AllocaIP)); - } + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = [&]() { + if (RequiresOuterTargetTask) + return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID, RTLoc, AllocaIP, + Dependencies, HasNoWait); + + return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID, + EmitTargetCallFallbackCB, KArgs, + DeviceID, RTLoc, AllocaIP); + }(); + + // Assume no error was returned because TaskBodyCB and + // EmitTargetCallFallbackCB don't produce any. The 'if' check enables + // accessing the returned value. + if (AfterIP) + Builder.restoreIP(*AfterIP); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget( const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo, ArrayRef NumTeams, ArrayRef NumThreads, @@ -7276,9 +7436,10 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget( // The target region is outlined into its own function. The LLVM IR for // the target region itself is generated using the callbacks CBFunc // and ArgAccessorFuncCB - emitTargetOutlinedFunction(*this, Builder, IsOffloadEntry, EntryInfo, - OutlinedFn, OutlinedFnID, Args, CBFunc, - ArgAccessorFuncCB); + if (Error Err = emitTargetOutlinedFunction( + *this, Builder, IsOffloadEntry, EntryInfo, OutlinedFn, OutlinedFnID, + Args, CBFunc, ArgAccessorFuncCB)) + return Err; // If we are not on the target device, then we need to generate code // to make a remote call (offload) to the previously outlined function @@ -7767,18 +7928,17 @@ void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn, Builder.SetInsertPoint(BB); } -void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, - BodyGenCallbackTy ElseGen, - InsertPointTy AllocaIP) { +Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, + BodyGenCallbackTy ElseGen, + InsertPointTy AllocaIP) { // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. if (auto *CI = dyn_cast(Cond)) { auto CondConstant = CI->getSExtValue(); if (CondConstant) - ThenGen(AllocaIP, Builder.saveIP()); - else - ElseGen(AllocaIP, Builder.saveIP()); - return; + return ThenGen(AllocaIP, Builder.saveIP()); + + return ElseGen(AllocaIP, Builder.saveIP()); } Function *CurFn = Builder.GetInsertBlock()->getParent(); @@ -7791,16 +7951,19 @@ void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, Builder.CreateCondBr(Cond, ThenBlock, ElseBlock); // Emit the 'then' code. emitBlock(ThenBlock, CurFn); - ThenGen(AllocaIP, Builder.saveIP()); + if (Error Err = ThenGen(AllocaIP, Builder.saveIP())) + return Err; emitBranch(ContBlock); // Emit the 'else' code if present. // There is no need to emit line number for unconditional branch. emitBlock(ElseBlock, CurFn); - ElseGen(AllocaIP, Builder.saveIP()); + if (Error Err = ElseGen(AllocaIP, Builder.saveIP())) + return Err; // There is no need to emit line number for unconditional branch. emitBranch(ContBlock); // Emit the continuation block for code after the if. emitBlock(ContBlock, CurFn, /*IsFinished=*/true); + return Error::success(); } bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic( @@ -7948,7 +8111,7 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc, return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate( const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) { @@ -7969,8 +8132,11 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate( "OpenMP atomic does not support LT or GT operations"); }); - emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp, - X.IsVolatile, IsXBinopExpr); + Expected> AtomicResult = + emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp, + X.IsVolatile, IsXBinopExpr); + if (!AtomicResult) + return AtomicResult.takeError(); checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update); return Builder.saveIP(); } @@ -8010,7 +8176,7 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2, llvm_unreachable("Unsupported atomic update operation"); } -std::pair OpenMPIRBuilder::emitAtomicUpdate( +Expected> OpenMPIRBuilder::emitAtomicUpdate( InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) { @@ -8072,7 +8238,10 @@ std::pair OpenMPIRBuilder::emitAtomicUpdate( llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2); PHI->addIncoming(AtomicLoadRes.first, CurBB); Value *OldExprVal = PHI; - Value *Upd = UpdateOp(OldExprVal, Builder); + Expected CBResult = UpdateOp(OldExprVal, Builder); + if (!CBResult) + return CBResult.takeError(); + Value *Upd = *CBResult; Builder.CreateStore(Upd, NewAtomicAddr); AtomicOrdering Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); @@ -8129,7 +8298,10 @@ std::pair OpenMPIRBuilder::emitAtomicUpdate( } } - Value *Upd = UpdateOp(OldExprVal, Builder); + Expected CBResult = UpdateOp(OldExprVal, Builder); + if (!CBResult) + return CBResult.takeError(); + Value *Upd = *CBResult; Builder.CreateStore(Upd, NewAtomicAddr); LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr); AtomicOrdering Failure = @@ -8158,7 +8330,7 @@ std::pair OpenMPIRBuilder::emitAtomicUpdate( return Res; } -OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture( +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture( const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, AtomicOpValue &V, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, @@ -8181,11 +8353,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture( // If UpdateExpr is 'x' updated with some `expr` not based on 'x', // 'x' is simply atomically rewritten with 'expr'. AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg); - std::pair Result = + Expected> AtomicResult = emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp, X.IsVolatile, IsXBinopExpr); - - Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second); + if (!AtomicResult) + return AtomicResult.takeError(); + Value *CapturedVal = + (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second); Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile); checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture); @@ -8380,7 +8554,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare( return Builder.saveIP(); } -OpenMPIRBuilder::InsertPointTy +OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower, Value *NumTeamsUpper, Value *ThreadLimit, @@ -8463,7 +8637,8 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc, // Generate the body of teams. InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin()); InsertPointTy CodeGenIP(BodyBB, BodyBB->begin()); - BodyGenCB(AllocaIP, CodeGenIP); + if (Error Err = BodyGenCB(AllocaIP, CodeGenIP)) + return Err; OutlineInfo OI; OI.EntryBB = AllocaBB; diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index 35664a5c7a2ac2..9e25620710fc84 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -1093,6 +1093,7 @@ struct OpenMPOpt { CGStartBB->getTerminator()->setSuccessor(0, StartBB); assert(EndBB != nullptr && "EndBB should not be null"); EndBB->getTerminator()->setSuccessor(0, CGEndBB); + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &, @@ -1101,7 +1102,7 @@ struct OpenMPOpt { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy CodeGenIP) {}; + auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); }; /// Create a sequential execution region within a merged parallel region, /// encapsulated in a master construct with a barrier for synchronization. @@ -1132,8 +1133,9 @@ struct OpenMPOpt { CGStartBB->getTerminator()->setSuccessor(0, SeqStartBB); assert(SeqEndBB != nullptr && "SeqEndBB should not be null"); SeqEndBB->getTerminator()->setSuccessor(0, CGEndBB); + return Error::success(); }; - auto FiniCB = [&](InsertPointTy CodeGenIP) {}; + auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); }; // Find outputs from the sequential region to outside users and // broadcast their values to them. @@ -1176,12 +1178,15 @@ struct OpenMPOpt { OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(ParentBB, ParentBB->end()), DL); - InsertPointTy SeqAfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy SeqAfterIP = OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB); + assert(SeqAfterIP && "Unexpected error creating master"); - OMPInfoCache.OMPBuilder.createBarrier(SeqAfterIP, OMPD_parallel); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierAfterIP = + OMPInfoCache.OMPBuilder.createBarrier(*SeqAfterIP, OMPD_parallel); + assert(BarrierAfterIP && "Unexpected error creating barrier"); - BranchInst::Create(SeqAfterBB, SeqAfterIP.getBlock()); + BranchInst::Create(SeqAfterBB, SeqAfterIP->getBlock()); LLVM_DEBUG(dbgs() << TAG << "After sequential inlining " << *OuterFn << "\n"); @@ -1251,10 +1256,12 @@ struct OpenMPOpt { OriginalFn->getEntryBlock().getFirstInsertionPt()); // Create the merged parallel region with default proc binding, to // avoid overriding binding settings, and without explicit cancellation. - InsertPointTy AfterIP = OMPInfoCache.OMPBuilder.createParallel( - Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, - OMP_PROC_BIND_default, /* IsCancellable */ false); - BranchInst::Create(AfterBB, AfterIP.getBlock()); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPInfoCache.OMPBuilder.createParallel( + Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, + OMP_PROC_BIND_default, /* IsCancellable */ false); + assert(AfterIP && "Unexpected error creating parallel"); + BranchInst::Create(AfterBB, AfterIP->getBlock()); // Perform the actual outlining. OMPInfoCache.OMPBuilder.finalize(OriginalFn); @@ -1290,10 +1297,12 @@ struct OpenMPOpt { if (CI != MergableCIs.back()) { // TODO: Remove barrier if the merged parallel region includes the // 'nowait' clause. - OMPInfoCache.OMPBuilder.createBarrier( - InsertPointTy(NewCI->getParent(), - NewCI->getNextNode()->getIterator()), - OMPD_parallel); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPInfoCache.OMPBuilder.createBarrier( + InsertPointTy(NewCI->getParent(), + NewCI->getNextNode()->getIterator()), + OMPD_parallel); + assert(AfterIP && "Unexpected error creating barrier"); } CI->eraseFromParent(); diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp index fe04cbbce12dcd..630cd03c688012 100644 --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -27,6 +27,20 @@ using namespace llvm; using namespace omp; +// Wrapper lambdas to allow using EXPECT*() macros inside of error-returning +// callbacks. +#define FINICB_WRAPPER(cb) \ + [&cb](InsertPointTy IP) -> Error { \ + cb(IP); \ + return Error::success(); \ + } + +#define BODYGENCB_WRAPPER(cb) \ + [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) -> Error { \ + cb(AllocaIP, CodeGenIP); \ + return Error::success(); \ + } + namespace { /// Create an instruction that uses the values in \p Values. We use "printf" @@ -218,9 +232,13 @@ class OpenMPIRBuilderTest : public testing::Test { CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC}); if (Call) *Call = CallInst; + + return Error::success(); }; - CanonicalLoopInfo *Loop = + Expected LoopResult = OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *Loop = *LoopResult; // Finalize the function. Builder.restoreIP(Loop->getAfterIP()); @@ -327,14 +345,18 @@ TEST_F(OpenMPIRBuilderTest, CreateBarrier) { IRBuilder<> Builder(BB); - OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1 = + OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for); + assert(BarrierIP1 && "unexpected error"); EXPECT_TRUE(M->global_empty()); EXPECT_EQ(M->size(), 1U); EXPECT_EQ(F->size(), 1U); EXPECT_EQ(BB->size(), 0U); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); - OMPBuilder.createBarrier(Loc, OMPD_for); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2 = + OMPBuilder.createBarrier(Loc, OMPD_for); + assert(BarrierIP2 && "unexpected error"); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 1U); @@ -372,13 +394,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) { ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; - OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); + OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); - auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel); - Builder.restoreIP(NewIP); + OpenMPIRBuilder::InsertPointOrErrorTy NewIP = + OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel); + assert(NewIP && "unexpected error"); + Builder.restoreIP(*NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 4U); EXPECT_EQ(F->size(), 4U); @@ -400,7 +424,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) { EXPECT_EQ(Cancel->getNumUses(), 1U); Instruction *CancelBBTI = Cancel->getParent()->getTerminator(); EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); - EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock()); + EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP->getBlock()); EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U); CallInst *GTID1 = dyn_cast(&CancelBBTI->getSuccessor(1)->front()); EXPECT_NE(GTID1, nullptr); @@ -439,13 +463,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) { ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; - OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); + OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); - auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel); - Builder.restoreIP(NewIP); + OpenMPIRBuilder::InsertPointOrErrorTy NewIP = + OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel); + assert(NewIP && "unexpected error"); + Builder.restoreIP(*NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 4U); EXPECT_EQ(F->size(), 7U); @@ -473,7 +499,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) { EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(), - NewIP.getBlock()); + NewIP->getBlock()); EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U); CallInst *GTID1 = dyn_cast(&CancelBBTI->getSuccessor(1)->front()); EXPECT_NE(GTID1, nullptr); @@ -512,13 +538,15 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) { ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; - OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); + OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); - auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for); - Builder.restoreIP(NewIP); + OpenMPIRBuilder::InsertPointOrErrorTy NewIP = + OMPBuilder.createBarrier(Loc, OMPD_for); + assert(NewIP && "unexpected error"); + Builder.restoreIP(*NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 4U); @@ -540,7 +568,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) { EXPECT_EQ(Barrier->getNumUses(), 1U); Instruction *BarrierBBTI = Barrier->getParent()->getTerminator(); EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U); - EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock()); + EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP->getBlock()); EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U); EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 1U); @@ -563,7 +591,9 @@ TEST_F(OpenMPIRBuilderTest, DbgLoc) { IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); - OMPBuilder.createBarrier(Loc, OMPD_for); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createBarrier(Loc, OMPD_for); + assert(AfterIP && "unexpected error"); CallInst *GTID = dyn_cast(&BB->front()); CallInst *Barrier = dyn_cast(GTID->getNextNode()); EXPECT_EQ(GTID->getDebugLoc(), DL); @@ -627,6 +657,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -654,19 +685,23 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { + ++NumFinalizationPoints; + return Error::success(); + }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 1U); EXPECT_EQ(NumFinalizationPoints, 1U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -735,6 +770,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -762,18 +798,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { + ++NumFinalizationPoints; + return Error::success(); + }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 1U); EXPECT_EQ(NumFinalizationPoints, 1U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -826,6 +866,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) { auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { ++NumInnerBodiesGenerated; + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -841,7 +882,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { + ++NumFinalizationPoints; + return Error::success(); + }; auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { ++NumOuterBodiesGenerated; @@ -849,27 +893,29 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) { BasicBlock *CGBB = CodeGenIP.getBlock(); BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint()); CGBB->getTerminator()->eraseFromParent(); - ; - IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel( InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateBr(NewBB); + return Error::success(); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumInnerBodiesGenerated, 1U); EXPECT_EQ(NumOuterBodiesGenerated, 1U); EXPECT_EQ(NumFinalizationPoints, 2U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -920,6 +966,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { ++NumInnerBodiesGenerated; + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -935,7 +982,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { + ++NumFinalizationPoints; + return Error::success(); + }; auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { ++NumOuterBodiesGenerated; @@ -948,32 +998,36 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { NewBB1->getTerminator()->eraseFromParent(); ; - IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1 = OMPBuilder.createParallel( InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP1 && "unexpected error"); - Builder.restoreIP(AfterIP1); + Builder.restoreIP(*AfterIP1); Builder.CreateBr(NewBB1); - IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2 = OMPBuilder.createParallel( InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP2 && "unexpected error"); - Builder.restoreIP(AfterIP2); + Builder.restoreIP(*AfterIP2); Builder.CreateBr(NewBB2); + return Error::success(); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumInnerBodiesGenerated, 2U); EXPECT_EQ(NumOuterBodiesGenerated, 1U); EXPECT_EQ(NumFinalizationPoints, 3U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -1043,6 +1097,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm, &ElseTerm); + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -1073,20 +1128,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; // No destructors. + return Error::success(); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, Builder.CreateIsNotNull(F->arg_begin()), nullptr, OMP_PROC_BIND_default, false); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 1U); EXPECT_EQ(NumFinalizationPoints, 1U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -1141,8 +1198,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { // Create three barriers, two cancel barriers but only one checked. Function *CBFn, *BFn; - Builder.restoreIP( - OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel)); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1 = + OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel); + assert(BarrierIP1 && "unexpected error"); + Builder.restoreIP(*BarrierIP1); CBFn = M->getFunction("__kmpc_cancel_barrier"); BFn = M->getFunction("__kmpc_barrier"); @@ -1153,8 +1212,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U); CheckedBarrier = cast(CBFn->user_back()); - Builder.restoreIP( - OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true)); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2 = + OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true); + assert(BarrierIP2 && "unexpected error"); + Builder.restoreIP(*BarrierIP2); CBFn = M->getFunction("__kmpc_cancel_barrier"); BFn = M->getFunction("__kmpc_barrier"); ASSERT_NE(CBFn, nullptr); @@ -1164,8 +1225,10 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { ASSERT_TRUE(isa(BFn->user_back())); ASSERT_EQ(BFn->user_back()->getNumUses(), 0U); - Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, - false, false)); + OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP3 = + OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, false, false); + assert(BarrierIP3 && "unexpected error"); + Builder.restoreIP(*BarrierIP3); ASSERT_EQ(CBFn->getNumUses(), 2U); ASSERT_EQ(BFn->getNumUses(), 1U); ASSERT_TRUE(CBFn->user_back() != CheckedBarrier); @@ -1190,21 +1253,23 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { Builder.restoreIP(IP); Builder.CreateCall(FakeDestructor, {Builder.getInt32(NumFinalizationPoints)}); + return Error::success(); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = - OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, - Builder.CreateIsNotNull(F->arg_begin()), - nullptr, OMP_PROC_BIND_default, true); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel( + Loc, AllocaIP, BODYGENCB_WRAPPER(BodyGenCB), PrivCB, FiniCB, + Builder.CreateIsNotNull(F->arg_begin()), nullptr, OMP_PROC_BIND_default, + true); + assert(AfterIP && "unexpected error"); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 0U); EXPECT_EQ(NumFinalizationPoints, 2U); EXPECT_EQ(FakeDestructor->getNumUses(), 2U); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -1269,20 +1334,22 @@ TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) { Builder.CreateCall(TakeI32PtrFunc, I32PtrVal); Builder.CreateCall(TakeStructFunc, StructVal); Builder.CreateCall(TakeStructPtrFunc, StructPtrVal); + return Error::success(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &, Value &Inner, Value *&ReplacementValue) { ReplacementValue = &Inner; return CodeGenIP; }; - auto FiniCB = [](InsertPointTy) {}; + auto FiniCB = [](InsertPointTy) { return Error::success(); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); - Builder.restoreIP(AfterIP); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -1312,10 +1379,13 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; - CanonicalLoopInfo *Loop = + Expected LoopResult = OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *Loop = *LoopResult; Builder.restoreIP(Loop->getAfterIP()); ReturnInst *RetInst = Builder.CreateRetVoid(); @@ -1367,10 +1437,14 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) { Value *StartVal = ConstantInt::get(LCTy, Start); Value *StopVal = ConstantInt::get(LCTy, Stop); Value *StepVal = ConstantInt::get(LCTy, Step); - auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {}; - CanonicalLoopInfo *Loop = + auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) { + return Error::success(); + }; + Expected LoopResult = OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal, StepVal, IsSigned, InclusiveStop); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *Loop = *LoopResult; Loop->assertOK(); Builder.restoreIP(Loop->getAfterIP()); Value *TripCount = Loop->getTripCount(); @@ -1463,16 +1537,22 @@ TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) { Value *InnerLC) { Builder.restoreIP(InnerCodeGenIP); Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC}); + return Error::success(); }; - InnerLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner"); + assert(LoopResult && "unexpected error"); + InnerLoop = *LoopResult; Builder.restoreIP(InnerLoop->getAfterIP()); InbetweenTrail = createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC}); + return Error::success(); }; - CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer"); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *OuterLoop = *LoopResult; // Finish the function. Builder.restoreIP(OuterLoop->getAfterIP()); @@ -1582,12 +1662,18 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) { // Add something that consumes the induction variables to the body. createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC}); + return Error::success(); }; - InnerLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner"); + assert(LoopResult && "unexpected error"); + InnerLoop = *LoopResult; + return Error::success(); }; - CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, OuterLoopBodyGenCB, TripCount, "outer"); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *OuterLoop = *LoopResult; // Finalize the function. Builder.restoreIP(OuterLoop->getAfterIP()); @@ -1682,14 +1768,20 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) { // Add something that consumes the induction variable to the body. Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC}); + return Error::success(); }; - InnerLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal, InnerStep, false, false, ComputeIP, "inner"); + assert(LoopResult && "unexpected error"); + InnerLoop = *LoopResult; + return Error::success(); }; - CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false, false, ComputeIP, "outer"); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *OuterLoop = *LoopResult; // Finalize the function Builder.restoreIP(OuterLoop->getAfterIP()); @@ -1793,10 +1885,14 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) { Value *StepVal = ConstantInt::get(LCTy, Step); // Generate a loop. - auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {}; - CanonicalLoopInfo *Loop = + auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) { + return Error::success(); + }; + Expected LoopResult = OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal, StepVal, IsSigned, InclusiveStop); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *Loop = *LoopResult; InsertPointTy AfterIP = Loop->getAfterIP(); // Tile the loop. @@ -2245,19 +2341,22 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkshareLoopTarget) { Value *StartVal = ConstantInt::get(LCTy, 10); Value *StopVal = ConstantInt::get(LCTy, 52); Value *StepVal = ConstantInt::get(LCTy, 2); - auto LoopBodyGen = [&](InsertPointTy, Value *) {}; + auto LoopBodyGen = [&](InsertPointTy, Value *) { return Error::success(); }; - CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, LoopBodyGen, StartVal, StopVal, StepVal, false, false); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *CLI = *LoopResult; BasicBlock *Preheader = CLI->getPreheader(); Value *TripCount = CLI->getTripCount(); Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); - IRBuilder<>::InsertPoint AfterIP = OMPBuilder.applyWorkshareLoop( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop( DL, CLI, AllocaIP, true, OMP_SCHEDULE_Static, nullptr, false, false, false, false, WorksharingLoopType::ForStaticLoop); - Builder.restoreIP(AfterIP); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -2306,11 +2405,15 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) { Value *StartVal = ConstantInt::get(LCTy, 10); Value *StopVal = ConstantInt::get(LCTy, 52); Value *StepVal = ConstantInt::get(LCTy, 2); - auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {}; + auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) { + return Error::success(); + }; - CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, LoopBodyGen, StartVal, StopVal, StepVal, /*IsSigned=*/false, /*InclusiveStop=*/false); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *CLI = *LoopResult; BasicBlock *Preheader = CLI->getPreheader(); BasicBlock *Body = CLI->getBody(); Value *IV = CLI->getIndVar(); @@ -2319,8 +2422,9 @@ TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) { Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); InsertPointTy AllocaIP = Builder.saveIP(); - OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true, - OMP_SCHEDULE_Static); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop( + DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static); + assert(AfterIP && "unexpected error"); BasicBlock *Cond = Body->getSinglePredecessor(); Instruction *Cmp = &*Cond->begin(); @@ -2412,8 +2516,9 @@ TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) { Value *ChunkSize = ConstantInt::get(LCTy, 5); InsertPointTy AllocaIP{&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()}; - OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true, - OMP_SCHEDULE_Static, ChunkSize); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.applyWorkshareLoop( + DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkSize); + assert(AfterIP && "unexpected error"); OMPBuilder.finalize(); EXPECT_FALSE(verifyModule(*M, &errs())); @@ -2500,11 +2605,15 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) { Value *StepVal = ConstantInt::get(LCTy, 2); Value *ChunkVal = (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize); - auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {}; + auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) { + return Error::success(); + }; - CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, LoopBodyGen, StartVal, StopVal, StepVal, /*IsSigned=*/false, /*InclusiveStop=*/false); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *CLI = *LoopResult; Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); InsertPointTy AllocaIP = Builder.saveIP(); @@ -2517,7 +2626,7 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) { BasicBlock *LatchBlock = CLI->getLatch(); Value *IV = CLI->getIndVar(); - InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop( + OpenMPIRBuilder::InsertPointOrErrorTy EndIP = OMPBuilder.applyWorkshareLoop( DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType), ChunkVal, /*Simd=*/false, (SchedType & omp::OMPScheduleType::ModifierMonotonic) == @@ -2525,10 +2634,11 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) { (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) == omp::OMPScheduleType::ModifierNonmonotonic, /*Ordered=*/false); + assert(EndIP && "unexpected error"); // The returned value should be the "after" point. - ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock()); - ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint()); + ASSERT_EQ(EndIP->getBlock(), AfterIP.getBlock()); + ASSERT_EQ(EndIP->getPoint(), AfterIP.getPoint()); auto AllocaIter = BB->begin(); ASSERT_GE(std::distance(BB->begin(), BB->end()), 4); @@ -2603,7 +2713,7 @@ TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) { EXPECT_EQ(NumCallsInExitBlock, 2u); // Add a termination to our block and check that it is internally consistent. - Builder.restoreIP(EndIP); + Builder.restoreIP(*EndIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_FALSE(verifyModule(*M, &errs())); @@ -2642,11 +2752,15 @@ TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) { Value *StopVal = ConstantInt::get(LCTy, 52); Value *StepVal = ConstantInt::get(LCTy, 2); Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize); - auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {}; + auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) { + return llvm::Error::success(); + }; - CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( + Expected LoopResult = OMPBuilder.createCanonicalLoop( Loc, LoopBodyGen, StartVal, StopVal, StepVal, /*IsSigned=*/false, /*InclusiveStop=*/false); + assert(LoopResult && "unexpected error"); + CanonicalLoopInfo *CLI = *LoopResult; Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); InsertPointTy AllocaIP = Builder.saveIP(); @@ -2658,14 +2772,15 @@ TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) { BasicBlock *LatchBlock = CLI->getLatch(); Value *IV = CLI->getIndVar(); - InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop( + OpenMPIRBuilder::InsertPointOrErrorTy EndIP = OMPBuilder.applyWorkshareLoop( DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal, /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false, /*HasOrderedClause=*/true); + assert(EndIP && "unexpected error"); // Add a termination to our block and check that it is internally consistent. - Builder.restoreIP(EndIP); + Builder.restoreIP(*EndIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_FALSE(verifyModule(*M, &errs())); @@ -2749,7 +2864,10 @@ TEST_F(OpenMPIRBuilderTest, MasterDirective) { EXPECT_NE(IPBB->end(), IP.getPoint()); }; - Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createMaster( + Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB)); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); @@ -2827,8 +2945,10 @@ TEST_F(OpenMPIRBuilderTest, MaskedDirective) { }; Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0); - Builder.restoreIP( - OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createMasked( + Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB), Filter); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); @@ -2893,8 +3013,11 @@ TEST_F(OpenMPIRBuilderTest, CriticalDirective) { }; BasicBlock *EntryBB = Builder.GetInsertBlock(); - Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB, - "testCRT", nullptr)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createCritical(Builder, BODYGENCB_WRAPPER(BodyGenCB), + FINICB_WRAPPER(FiniCB), "testCRT", nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); CallInst *CriticalEntryCI = nullptr; for (auto &EI : *EntryBB) { @@ -3141,8 +3264,11 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) { // Test for "#omp ordered [threads]" BasicBlock *EntryBB = Builder.GetInsertBlock(); - Builder.restoreIP( - OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB), + FINICB_WRAPPER(FiniCB), true); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -3212,8 +3338,11 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) { // Test for "#omp ordered simd" BasicBlock *EntryBB = Builder.GetInsertBlock(); - Builder.restoreIP( - OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB), + FINICB_WRAPPER(FiniCB), false); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -3326,8 +3455,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirective) { EXPECT_NE(IPBB->end(), IP.getPoint()); }; - Builder.restoreIP( - OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, /*IsNowait*/ false)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB), + FINICB_WRAPPER(FiniCB), /*IsNowait*/ false); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); @@ -3416,8 +3548,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) { EXPECT_NE(IPBB->end(), IP.getPoint()); }; - Builder.restoreIP( - OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, /*IsNowait*/ true)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB), + FINICB_WRAPPER(FiniCB), /*IsNowait*/ true); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); @@ -3535,9 +3670,11 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveCopyPrivate) { EXPECT_NE(IPBB->end(), IP.getPoint()); }; - Builder.restoreIP(OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, - /*IsNowait*/ false, {CPVar}, - {CopyFunc})); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSingle( + Builder, BODYGENCB_WRAPPER(BodyGenCB), FINICB_WRAPPER(FiniCB), + /*IsNowait*/ false, {CPVar}, {CopyFunc}); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); @@ -3798,8 +3935,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) { Sub = IRB.CreateSub(ConstVal, Atomic); return Sub; }; - Builder.restoreIP(OMPBuilder.createAtomicUpdate( - Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate( + Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); BasicBlock *ContBB = EntryBB->getSingleSuccessor(); BranchInst *ContTI = dyn_cast(ContBB->getTerminator()); EXPECT_NE(ContTI, nullptr); @@ -3865,8 +4004,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) { Sub = IRB.CreateFSub(ConstVal, Atomic); return Sub; }; - Builder.restoreIP(OMPBuilder.createAtomicUpdate( - Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate( + Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); BasicBlock *ContBB = EntryBB->getSingleSuccessor(); BranchInst *ContTI = dyn_cast(ContBB->getTerminator()); EXPECT_NE(ContTI, nullptr); @@ -3931,8 +4072,10 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) { Sub = IRB.CreateSub(ConstVal, Atomic); return Sub; }; - Builder.restoreIP(OMPBuilder.createAtomicUpdate( - Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createAtomicUpdate( + Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); BasicBlock *ContBB = EntryBB->getSingleSuccessor(); BranchInst *ContTI = dyn_cast(ContBB->getTerminator()); EXPECT_NE(ContTI, nullptr); @@ -4003,9 +4146,12 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) { // integer update - not used auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; }; - Builder.restoreIP(OMPBuilder.createAtomicCapture( - Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr, - IsPostfixUpdate, IsXLHSInRHSPart)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createAtomicCapture(Builder, AllocaIP, X, V, Expr, AO, RMWOp, + UpdateOp, UpdateExpr, IsPostfixUpdate, + IsXLHSInRHSPart); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); EXPECT_EQ(EntryBB->getParent()->size(), 1U); AtomicRMWInst *ARWM = dyn_cast(Init->getNextNode()); EXPECT_NE(ARWM, nullptr); @@ -4361,12 +4507,15 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); - Builder.restoreIP(OMPBuilder.createTeams( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams( Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr, - /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr)); + /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -4423,14 +4572,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; // `F` has an argument - an integer, so we use that as the thread limit. - Builder.restoreIP(OMPBuilder.createTeams(/*=*/Builder, BodyGenCB, - /*NumTeamsLower=*/nullptr, - /*NumTeamsUpper=*/nullptr, - /*ThreadLimit=*/F->arg_begin(), - /*IfExpr=*/nullptr)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams( + /*=*/Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, + /*NumTeamsUpper=*/nullptr, /*ThreadLimit=*/F->arg_begin(), + /*IfExpr=*/nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4474,15 +4625,19 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; // `F` already has an integer argument, so we use that as upper bound to // `num_teams` - Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB, - /*NumTeamsLower=*/nullptr, - /*NumTeamsUpper=*/F->arg_begin(), - /*ThreadLimit=*/nullptr, - /*IfExpr=*/nullptr)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createTeams(Builder, BodyGenCB, + /*NumTeamsLower=*/nullptr, + /*NumTeamsUpper=*/F->arg_begin(), + /*ThreadLimit=*/nullptr, + /*IfExpr=*/nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4531,13 +4686,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; // `F` already has an integer argument, so we use that as upper bound to // `num_teams` - Builder.restoreIP( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, - /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr)); + /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4593,11 +4751,14 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); - Builder.restoreIP(OMPBuilder.createTeams( - Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams( + Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4644,13 +4805,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; // `F` already has an integer argument, so we use that as upper bound to // `num_teams` - Builder.restoreIP(OMPBuilder.createTeams( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams( Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr, - /*ThreadLimit=*/nullptr, IfExpr)); + /*ThreadLimit=*/nullptr, IfExpr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4707,12 +4871,15 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) { auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); Builder.CreateCall(FakeFunction, {}); + return Error::success(); }; // `F` already has an integer argument, so we use that as upper bound to // `num_teams` - Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, - NumTeamsUpper, ThreadLimit, IfExpr)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTeams( + Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, IfExpr); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -4937,6 +5104,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) { BodyIP = Builder.saveIP(); BodyAllocaIP = InnerAllocaIP; + return Error::success(); }; // Privatization for reduction creates local copies of reduction variables and @@ -4969,14 +5137,15 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) { }; // Do nothing in finalization. - auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); }; - InsertPointTy AfterIP = + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB, /* IfCondition */ nullptr, /* NumThreads */ nullptr, OMP_PROC_BIND_default, /* IsCancellable */ false); - Builder.restoreIP(AfterIP); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OpenMPIRBuilder::ReductionInfo ReductionInfos[] = { {SumType, SumReduced, SumPrivatized, @@ -4989,10 +5158,12 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) { bool ReduceVariableByRef[] = {false, false}; - OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos, - ReduceVariableByRef); + OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP = + OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos, + ReduceVariableByRef); + assert(ReductionsIP && "unexpected error"); - Builder.restoreIP(AfterIP); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(F); @@ -5172,6 +5343,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) { FirstBodyIP = Builder.saveIP(); FirstBodyAllocaIP = InnerAllocaIP; + return Error::success(); }; InsertPointTy SecondBodyIP, SecondBodyAllocaIP; @@ -5190,6 +5362,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) { SecondBodyIP = Builder.saveIP(); SecondBodyAllocaIP = InnerAllocaIP; + return Error::success(); }; // Privatization for reduction creates local copies of reduction variables and @@ -5224,36 +5397,44 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) { }; // Do nothing in finalization. - auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; }; + auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); }; - Builder.restoreIP( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1 = OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB, FiniCB, /* IfCondition */ nullptr, /* NumThreads */ nullptr, OMP_PROC_BIND_default, - /* IsCancellable */ false)); - InsertPointTy AfterIP = OMPBuilder.createParallel( + /* IsCancellable */ false); + assert(AfterIP1 && "unexpected error"); + Builder.restoreIP(*AfterIP1); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2 = OMPBuilder.createParallel( {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB, /* IfCondition */ nullptr, /* NumThreads */ nullptr, OMP_PROC_BIND_default, /* IsCancellable */ false); + assert(AfterIP2 && "unexpected error"); + Builder.restoreIP(*AfterIP2); OMPBuilder.Config.setIsGPU(false); bool ReduceVariableByRef[] = {false}; - OMPBuilder.createReductions( - FirstBodyIP, FirstBodyAllocaIP, - {{SumType, SumReduced, SumPrivatized, - /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction, - /*ReductionGenClang=*/nullptr, sumAtomicReduction}}, - ReduceVariableByRef); - OMPBuilder.createReductions( - SecondBodyIP, SecondBodyAllocaIP, - {{XorType, XorReduced, XorPrivatized, - /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction, - /*ReductionGenClang=*/nullptr, xorAtomicReduction}}, - ReduceVariableByRef); - - Builder.restoreIP(AfterIP); + OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP1 = + OMPBuilder.createReductions( + FirstBodyIP, FirstBodyAllocaIP, + {{SumType, SumReduced, SumPrivatized, + /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction, + /*ReductionGenClang=*/nullptr, sumAtomicReduction}}, + ReduceVariableByRef); + assert(ReductionsIP1 && "unexpected error"); + OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP2 = + OMPBuilder.createReductions( + SecondBodyIP, SecondBodyAllocaIP, + {{XorType, XorReduced, XorPrivatized, + /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction, + /*ReductionGenClang=*/nullptr, xorAtomicReduction}}, + ReduceVariableByRef); + assert(ReductionsIP2 && "unexpected error"); + + Builder.restoreIP(*AfterIP2); Builder.CreateRetVoid(); OMPBuilder.finalize(F); @@ -5320,8 +5501,10 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) { llvm::SmallVector SectionCBVector; llvm::SmallVector CaseBBs; - auto FiniCB = [&](InsertPointTy IP) {}; - auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto FiniCB = [&](InsertPointTy IP) { return Error::success(); }; + auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; SectionCBVector.push_back(SectionCB); auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, @@ -5329,8 +5512,10 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) { llvm::Value *&ReplVal) { return CodeGenIP; }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, - PrivCB, FiniCB, false, false)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSections( + Loc, AllocaIP, SectionCBVector, PrivCB, FiniCB, false, false); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); // Required at the end of the function EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr); EXPECT_FALSE(verifyModule(*M, &errs())); @@ -5371,6 +5556,7 @@ TEST_F(OpenMPIRBuilderTest, CreateSections) { Value *PrivLoad = Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca"); Builder.CreateICmpNE(F->arg_begin(), PrivLoad); + return Error::success(); }; auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) { @@ -5383,8 +5569,11 @@ TEST_F(OpenMPIRBuilderTest, CreateSections) { IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); - Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, - PrivCB, FiniCB, false, false)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, PrivCB, + FINICB_WRAPPER(FiniCB), false, false); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); // Required at the end of the function // Switch BB's predecessor is loop condition BB, whose successor at index 1 is @@ -5468,10 +5657,12 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) { auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) { return CodeGenIP; }; - auto FiniCB = [&](InsertPointTy IP) {}; + auto FiniCB = [&](InsertPointTy IP) { return Error::success(); }; - Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, - PrivCB, FiniCB, false, true)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createSections( + Loc, AllocaIP, SectionCBVector, PrivCB, FiniCB, false, true); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); // Required at the end of the function for (auto &Inst : instructions(*F)) { EXPECT_FALSE(isa(Inst) && @@ -5692,9 +5883,11 @@ TEST_F(OpenMPIRBuilderTest, TargetEnterData) { OMPBuilder.Config.setIsGPU(true); llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper; - Builder.restoreIP(OMPBuilder.createTargetData( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTargetData( Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), - /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc)); + /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); CallInst *TargetDataCall = dyn_cast(&BB->back()); EXPECT_NE(TargetDataCall, nullptr); @@ -5751,9 +5944,11 @@ TEST_F(OpenMPIRBuilderTest, TargetExitData) { OMPBuilder.Config.setIsGPU(true); llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper; - Builder.restoreIP(OMPBuilder.createTargetData( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTargetData( Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), - /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc)); + /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); CallInst *TargetDataCall = dyn_cast(&BB->back()); EXPECT_NE(TargetDataCall, nullptr); @@ -5859,9 +6054,12 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) { return Builder.saveIP(); }; - Builder.restoreIP(OMPBuilder.createTargetData( - Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), - /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB)); + OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP1 = + OMPBuilder.createTargetData( + Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), + /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB); + assert(TargetDataIP1 && "unexpected error"); + Builder.restoreIP(*TargetDataIP1); CallInst *TargetDataCall = dyn_cast(&BB->back()); EXPECT_NE(TargetDataCall, nullptr); @@ -5884,9 +6082,12 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) { EXPECT_EQ(TargetDataCall, nullptr); return Builder.saveIP(); }; - Builder.restoreIP(OMPBuilder.createTargetData( - Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), - /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyTargetCB)); + OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP2 = + OMPBuilder.createTargetData( + Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID), + /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyTargetCB); + assert(TargetDataIP2 && "unexpected error"); + Builder.restoreIP(*TargetDataIP2); EXPECT_TRUE(CheckDevicePassBodyGen); Builder.CreateRetVoid(); @@ -5981,9 +6182,11 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) { TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17); OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL}); - Builder.restoreIP(OMPBuilder.createTarget( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTarget( OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(), Builder.saveIP(), - EntryInfo, -1, 0, Inputs, GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB)); + EntryInfo, -1, 0, Inputs, GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6089,11 +6292,13 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) { TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2, /*Line=*/3, /*Count=*/0); - Builder.restoreIP( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP, EntryInfo, /*NumTeams=*/-1, /*NumThreads=*/0, CapturedArgs, GenMapInfoCB, - BodyGenCB, SimpleArgAccessorCB)); + BodyGenCB, SimpleArgAccessorCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -6238,11 +6443,13 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) { TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2, /*Line=*/3, /*Count=*/0); - Builder.restoreIP( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP, EntryInfo, /*NumTeams=*/-1, /*NumThreads=*/0, CapturedArgs, GenMapInfoCB, - BodyGenCB, SimpleArgAccessorCB)); + BodyGenCB, SimpleArgAccessorCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); @@ -6354,15 +6561,17 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) { Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL); - Builder.restoreIP(OMPBuilder.createTask( - Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), - BodyGenCB)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask( + Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6460,15 +6669,18 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) { F->setName("func"); IRBuilder<> Builder(BB); - auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL); - Builder.restoreIP(OMPBuilder.createTask( - Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), - BodyGenCB)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask( + Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6490,14 +6702,18 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) { OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); - auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL); - Builder.restoreIP(OMPBuilder.createTask( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask( Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB, - /*Tied=*/false)); + /*Tied=*/false); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6520,7 +6736,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) { OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); - auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( @@ -6532,9 +6750,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) { Type::getInt32Ty(M->getContext()), InDep); DDS.push_back(DDIn); } - Builder.restoreIP(OMPBuilder.createTask( + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTask( Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB, - /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS)); + /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6594,7 +6814,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) { OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); - auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP(); Builder.SetInsertPoint(BodyBB); @@ -6602,8 +6824,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) { CmpInst::Predicate::ICMP_EQ, F->getArg(0), ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U)); OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL); - Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB, - /*Tied=*/false, Final)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB, + /*Tied=*/false, Final); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6648,7 +6873,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) { OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); - auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {}; + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + return Error::success(); + }; BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP(); Builder.SetInsertPoint(BodyBB); @@ -6656,9 +6883,11 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) { CmpInst::Predicate::ICMP_EQ, F->getArg(0), ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U)); OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL); - Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB, - /*Tied=*/false, /*Final=*/nullptr, - IfCondition)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = + OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB, + /*Tied=*/false, /*Final=*/nullptr, IfCondition); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6742,15 +6971,17 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) { SplitBlockAndInsertIfThenElse(InternalIfCmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); + return Error::success(); }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL); - Builder.restoreIP(OMPBuilder.createTaskgroup( - Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), - BodyGenCB)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTaskgroup( + Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); @@ -6823,9 +7054,13 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) { Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64); Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64)); Builder.CreateStore(AddInst, Alloca64); + return Error::success(); }; OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL); - Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1)); + OpenMPIRBuilder::InsertPointOrErrorTy TaskIP1 = + OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1); + assert(TaskIP1 && "unexpected error"); + Builder.restoreIP(*TaskIP1); auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { Builder.restoreIP(CodeGenIP); @@ -6833,18 +7068,24 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) { Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32); Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32)); Builder.CreateStore(AddInst, Alloca32); + return Error::success(); }; OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL); - Builder.restoreIP(OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2)); + OpenMPIRBuilder::InsertPointOrErrorTy TaskIP2 = + OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2); + assert(TaskIP2 && "unexpected error"); + Builder.restoreIP(*TaskIP2); + return Error::success(); }; BasicBlock *AllocaBB = Builder.GetInsertBlock(); BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split"); OpenMPIRBuilder::LocationDescription Loc( InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL); - Builder.restoreIP(OMPBuilder.createTaskgroup( - Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), - BodyGenCB)); + OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = OMPBuilder.createTaskgroup( + Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB); + assert(AfterIP && "unexpected error"); + Builder.restoreIP(*AfterIP); OMPBuilder.finalize(); Builder.CreateRetVoid(); diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 27cd38dc3c62d9..fc2f88b766f1c5 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -137,9 +137,9 @@ findAllocaInsertPoint(llvm::IRBuilderBase &builder, /// region, and a branch from any block with an successor-less OpenMP terminator /// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes /// of the continuation block if provided. -static llvm::BasicBlock *convertOmpOpRegions( +static llvm::Expected convertOmpOpRegions( Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, + LLVM::ModuleTranslation &moduleTranslation, SmallVectorImpl *continuationBlockPHIs = nullptr) { llvm::BasicBlock *continuationBlock = splitBB(builder, true, "omp.region.cont"); @@ -215,10 +215,8 @@ static llvm::BasicBlock *convertOmpOpRegions( llvm::IRBuilderBase::InsertPointGuard guard(builder); if (failed( - moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder))) { - bodyGenStatus = failure(); - return continuationBlock; - } + moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder))) + return llvm::createStringError("failed region translation"); // Special handling for `omp.yield` and `omp.terminator` (we may have more // than one): they return the control to the parent OpenMP dialect operation @@ -270,21 +268,19 @@ convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { auto maskedOp = cast(opInst); using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. - LogicalResult bodyGenStatus = success(); auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { // MaskedOp has only one region associated with it. auto ®ion = maskedOp.getRegion(); builder.restoreIP(codeGenIP); - convertOmpOpRegions(region, "omp.masked.region", builder, moduleTranslation, - bodyGenStatus); + return convertOmpOpRegions(region, "omp.masked.region", builder, + moduleTranslation) + .takeError(); }; // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; llvm::Value *filterVal = nullptr; if (auto filterVar = maskedOp.getFilteredThreadId()) { @@ -296,8 +292,14 @@ convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder, } assert(filterVal != nullptr); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMasked( - ompLoc, bodyGenCB, finiCB, filterVal)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createMasked(ompLoc, bodyGenCB, + finiCB, filterVal); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); return success(); } @@ -306,25 +308,28 @@ static LogicalResult convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. - LogicalResult bodyGenStatus = success(); - auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { // MasterOp has only one region associated with it. auto ®ion = cast(opInst).getRegion(); builder.restoreIP(codeGenIP); - convertOmpOpRegions(region, "omp.master.region", builder, moduleTranslation, - bodyGenStatus); + return convertOmpOpRegions(region, "omp.master.region", builder, + moduleTranslation) + .takeError(); }; // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMaster( - ompLoc, bodyGenCB, finiCB)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createMaster(ompLoc, bodyGenCB, + finiCB); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); return success(); } @@ -334,21 +339,19 @@ convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; auto criticalOp = cast(opInst); - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. - LogicalResult bodyGenStatus = success(); auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { // CriticalOp has only one region associated with it. auto ®ion = cast(opInst).getRegion(); builder.restoreIP(codeGenIP); - convertOmpOpRegions(region, "omp.critical.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(region, "omp.critical.region", builder, + moduleTranslation) + .takeError(); }; // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); @@ -366,8 +369,14 @@ convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext), static_cast(criticalDeclareOp.getHint())); } - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createCritical( - ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint)); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createCritical( + ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); return success(); } @@ -468,27 +477,30 @@ static LogicalResult inlineConvertOmpRegions( return success(); } - LogicalResult bodyGenStatus = success(); SmallVector phis; - llvm::BasicBlock *continuationBlock = convertOmpOpRegions( - region, blockName, builder, moduleTranslation, bodyGenStatus, &phis); - if (failed(bodyGenStatus)) - return failure(); + llvm::Expected continuationBlock = + convertOmpOpRegions(region, blockName, builder, moduleTranslation, &phis); + + if (!continuationBlock) + return region.getParentOp()->emitError( + llvm::toString(continuationBlock.takeError())); + if (continuationBlockArgs) llvm::append_range(*continuationBlockArgs, phis); - builder.SetInsertPoint(continuationBlock, - continuationBlock->getFirstInsertionPt()); + builder.SetInsertPoint(*continuationBlock, + (*continuationBlock)->getFirstInsertionPt()); return success(); } namespace { /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to /// store lambdas with capture. -using OwningReductionGen = std::function; +using OwningReductionGen = + std::function; using OwningAtomicReductionGen = - std::function; } // namespace @@ -505,19 +517,20 @@ makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, OwningReductionGen gen = [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Value *lhs, llvm::Value *rhs, - llvm::Value *&result) mutable { - moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs); - moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs); - builder.restoreIP(insertPoint); - SmallVector phis; - if (failed(inlineConvertOmpRegions(decl.getReductionRegion(), - "omp.reduction.nonatomic.body", - builder, moduleTranslation, &phis))) - return llvm::OpenMPIRBuilder::InsertPointTy(); - assert(phis.size() == 1); - result = phis[0]; - return builder.saveIP(); - }; + llvm::Value *&result) mutable + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { + moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs); + moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs); + builder.restoreIP(insertPoint); + SmallVector phis; + if (failed(inlineConvertOmpRegions(decl.getReductionRegion(), + "omp.reduction.nonatomic.body", builder, + moduleTranslation, &phis))) + return llvm::createStringError("failed reduction region translation"); + assert(phis.size() == 1); + result = phis[0]; + return builder.saveIP(); + }; return gen; } @@ -537,18 +550,19 @@ makeAtomicReductionGen(omp::DeclareReductionOp decl, // avoid the dangling reference after the parent function returns. OwningAtomicReductionGen atomicGen = [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *, - llvm::Value *lhs, llvm::Value *rhs) mutable { - moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs); - moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs); - builder.restoreIP(insertPoint); - SmallVector phis; - if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(), - "omp.reduction.atomic.body", builder, - moduleTranslation, &phis))) - return llvm::OpenMPIRBuilder::InsertPointTy(); - assert(phis.empty()); - return builder.saveIP(); - }; + llvm::Value *lhs, llvm::Value *rhs) mutable + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { + moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs); + moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs); + builder.restoreIP(insertPoint); + SmallVector phis; + if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(), + "omp.reduction.atomic.body", builder, + moduleTranslation, &phis))) + return llvm::createStringError("failed reduction region translation"); + assert(phis.empty()); + return builder.saveIP(); + }; return atomicGen; } @@ -593,27 +607,29 @@ convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, if (orderedRegionOp.getParLevelSimd()) return failure(); - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. - LogicalResult bodyGenStatus = success(); - auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { // OrderedOp has only one region associated with it. auto ®ion = cast(opInst).getRegion(); builder.restoreIP(codeGenIP); - convertOmpOpRegions(region, "omp.ordered.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(region, "omp.ordered.region", builder, + moduleTranslation) + .takeError(); }; // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP( + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = moduleTranslation.getOpenMPBuilder()->createOrderedThreadsSimd( - ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd())); - return bodyGenStatus; + ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd()); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } namespace { @@ -811,15 +827,24 @@ static LogicalResult createReductionsAndCleanup( // and remove it later. llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable(); builder.SetInsertPoint(tempTerminator); - llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint = + llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint = ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos, isByRef, op.getNowait()); - if (!contInsertPoint.getBlock()) + + if (!contInsertPoint) + return op.emitError(llvm::toString(contInsertPoint.takeError())); + + if (!contInsertPoint->getBlock()) return op->emitOpError() << "failed to convert reductions"; - auto nextInsertionPoint = - ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for); + + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + ompBuilder->createBarrier(*contInsertPoint, llvm::omp::OMPD_for); + + if (!afterIP) + return op.emitError(llvm::toString(afterIP.takeError())); + tempTerminator->eraseFromParent(); - builder.restoreIP(nextInsertionPoint); + builder.restoreIP(*afterIP); // after the construct, deallocate private reduction variables SmallVector reductionRegions; @@ -958,7 +983,6 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation::SaveStack mappingGuard( moduleTranslation, reductionVariableMap); - LogicalResult bodyGenStatus = success(); SmallVector sectionCBs; for (Operation &op : *sectionsOp.getRegion().begin()) { @@ -967,9 +991,8 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, continue; Region ®ion = sectionOp.getRegion(); - auto sectionCB = [§ionsOp, ®ion, &builder, &moduleTranslation, - &bodyGenStatus](InsertPointTy allocaIP, - InsertPointTy codeGenIP) { + auto sectionCB = [§ionsOp, ®ion, &builder, &moduleTranslation]( + InsertPointTy allocaIP, InsertPointTy codeGenIP) { builder.restoreIP(codeGenIP); // map the omp.section reduction block argument to the omp.sections block @@ -985,8 +1008,9 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, moduleTranslation.mapValue(sectionArg, llvmVal); } - convertOmpOpRegions(region, "omp.section.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(region, "omp.section.region", builder, + moduleTranslation) + .takeError(); }; sectionCBs.push_back(sectionCB); } @@ -1003,24 +1027,27 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, // attribute (shared, private, firstprivate, ...) of variables. // Currently defaults to shared. auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &, - llvm::Value &vPtr, - llvm::Value *&replacementValue) -> InsertPointTy { + llvm::Value &vPtr, llvm::Value *&replacementValue) + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { replacementValue = &vPtr; return codeGenIP; }; // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; allocaIP = findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSections( - ompLoc, allocaIP, sectionCBs, privCB, finiCB, false, - sectionsOp.getNowait())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createSections( + ompLoc, allocaIP, sectionCBs, privCB, finiCB, false, + sectionsOp.getNowait()); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); - if (failed(bodyGenStatus)) - return bodyGenStatus; + builder.restoreIP(*afterIP); // Process the reductions if required. return createReductionsAndCleanup(sectionsOp, builder, moduleTranslation, @@ -1034,16 +1061,17 @@ convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - LogicalResult bodyGenStatus = success(); + if (!singleOp.getPrivateVars().empty() || singleOp.getPrivateSyms()) return singleOp.emitError("unhandled clauses for translation to LLVM IR"); auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) { builder.restoreIP(codegenIP); - convertOmpOpRegions(singleOp.getRegion(), "omp.single.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(singleOp.getRegion(), "omp.single.region", + builder, moduleTranslation) + .takeError(); }; - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; // Handle copyprivate Operation::operand_range cpVars = singleOp.getCopyprivateVars(); @@ -1058,9 +1086,16 @@ convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, moduleTranslation.lookupFunction(llvmFuncOp.getName())); } - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSingle( - ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars, llvmCPFuncs)); - return bodyGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createSingle( + ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars, + llvmCPFuncs); + + if (!afterIP) + return singleOp.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } // Convert an OpenMP Teams construct to LLVM IR using OpenMPIRBuilder @@ -1068,7 +1103,6 @@ static LogicalResult convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - LogicalResult bodyGenStatus = success(); if (!op.getAllocatorVars().empty() || op.getReductionSyms() || !op.getPrivateVars().empty() || op.getPrivateSyms()) return op.emitError("unhandled clauses for translation to LLVM IR"); @@ -1077,8 +1111,9 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation::SaveStack frame( moduleTranslation, allocaIP); builder.restoreIP(codegenIP); - convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder, + moduleTranslation) + .takeError(); }; llvm::Value *numTeamsLower = nullptr; @@ -1098,9 +1133,15 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, ifExpr = moduleTranslation.lookupValue(ifVar); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTeams( - ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr)); - return bodyGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createTeams( + ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr); + + if (!afterIP) + return op.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } static void @@ -1134,7 +1175,6 @@ static LogicalResult convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - LogicalResult bodyGenStatus = success(); if (taskOp.getUntiedAttr() || taskOp.getMergeableAttr() || taskOp.getInReductionSyms() || taskOp.getPriority() || !taskOp.getAllocateVars().empty() || !taskOp.getPrivateVars().empty() || @@ -1148,8 +1188,9 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, moduleTranslation, allocaIP); builder.restoreIP(codegenIP); - convertOmpOpRegions(taskOp.getRegion(), "omp.task.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(taskOp.getRegion(), "omp.task.region", builder, + moduleTranslation) + .takeError(); }; SmallVector dds; @@ -1159,11 +1200,17 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTask( - ompLoc, allocaIP, bodyCB, !taskOp.getUntied(), - moduleTranslation.lookupValue(taskOp.getFinal()), - moduleTranslation.lookupValue(taskOp.getIfExpr()), dds)); - return bodyGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createTask( + ompLoc, allocaIP, bodyCB, !taskOp.getUntied(), + moduleTranslation.lookupValue(taskOp.getFinal()), + moduleTranslation.lookupValue(taskOp.getIfExpr()), dds); + + if (!afterIP) + return taskOp.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } /// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder. @@ -1171,20 +1218,27 @@ static LogicalResult convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - LogicalResult bodyGenStatus = success(); - if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) { + if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) return tgOp.emitError("unhandled clauses for translation to LLVM IR"); - } + auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) { builder.restoreIP(codegenIP); - convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region", + builder, moduleTranslation) + .takeError(); }; + InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTaskgroup( - ompLoc, allocaIP, bodyCB)); - return bodyGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + moduleTranslation.getOpenMPBuilder()->createTaskgroup(ompLoc, allocaIP, + bodyCB); + + if (!afterIP) + return tgOp.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } static LogicalResult @@ -1258,12 +1312,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); // Generator of the canonical loop body. - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. SmallVector loopInfos; SmallVector bodyInsertPoints; - LogicalResult bodyGenStatus = success(); - auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) { + auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, + llvm::Value *iv) -> llvm::Error { // Make sure further conversions know about the induction variable. moduleTranslation.mapValue( loopOp.getRegion().front().getArgument(loopInfos.size()), iv); @@ -1274,12 +1326,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, bodyInsertPoints.push_back(ip); if (loopInfos.size() != loopOp.getNumLoops() - 1) - return; + return llvm::Error::success(); // Convert the body of the loop. builder.restoreIP(ip); - convertOmpOpRegions(loopOp.getRegion(), "omp.wsloop.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(loopOp.getRegion(), "omp.wsloop.region", builder, + moduleTranslation) + .takeError(); }; // Delegate actual loop construction to the OpenMP IRBuilder. @@ -1304,12 +1357,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back()); computeIP = loopInfos.front()->getPreheaderIP(); } - loopInfos.push_back(ompBuilder->createCanonicalLoop( - loc, bodyGen, lowerBound, upperBound, step, - /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP)); - if (failed(bodyGenStatus)) - return failure(); + llvm::Expected result = + ompBuilder->createCanonicalLoop( + loc, bodyGen, lowerBound, upperBound, step, + /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP); + + if (!result) + return loopOp.emitError(llvm::toString(result.takeError())); + + loopInfos.push_back(*result); } // Collapse loops. Store the insertion point because LoopInfos may get @@ -1325,11 +1382,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, std::optional scheduleMod = wsloopOp.getScheduleMod(); bool isSimd = wsloopOp.getScheduleSimd(); - ompBuilder->applyWorkshareLoop( - ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(), - convertToScheduleKind(schedule), chunk, isSimd, - scheduleMod == omp::ScheduleModifier::monotonic, - scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP = + ompBuilder->applyWorkshareLoop( + ompLoc.DL, loopInfo, allocaIP, !wsloopOp.getNowait(), + convertToScheduleKind(schedule), chunk, isSimd, + scheduleMod == omp::ScheduleModifier::monotonic, + scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered); + + if (!wsloopIP) + return opInst.emitError(llvm::toString(wsloopIP.takeError())); // Continue building IR after the loop. Note that the LoopInfo returned by // `collapseLoops` points inside the outermost loop and is intended for @@ -1350,10 +1411,6 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; ArrayRef isByRef = getIsByRef(opInst.getReductionByref()); assert(isByRef.size() == opInst.getNumReductionVars()); - - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. - LogicalResult bodyGenStatus = success(); llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); // Collect delayed privatization declarations @@ -1372,7 +1429,8 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, opInst.getNumReductionVars()); SmallVector deferredStores; - auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { + auto bodyGenCB = [&](InsertPointTy allocaIP, + InsertPointTy codeGenIP) -> llvm::Error { // Allocate private vars llvm::BranchInst *allocaTerminator = llvm::cast(allocaIP.getBlock()->getTerminator()); @@ -1418,10 +1476,11 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, builder.SetInsertPoint(privAllocBlock->getTerminator()); } if (failed(inlineConvertOmpRegions(allocRegion, "omp.private.alloc", - builder, moduleTranslation, &phis))) { - bodyGenStatus = failure(); - return; - } + builder, moduleTranslation, &phis))) + return llvm::createStringError( + "failed to inline `alloc` region of an `omp.private` op in the " + "parallel region"); + assert(phis.size() == 1 && "expected one allocation to be yielded"); moduleTranslation.mapValue(privateBlockArgs[i], phis[0]); @@ -1447,7 +1506,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, opInst, reductionArgs, builder, moduleTranslation, allocaIP, reductionDecls, privateReductionVariables, reductionVariableMap, deferredStores, isByRef))) - bodyGenStatus = failure(); + return llvm::createStringError("failed reduction vars allocation"); // Apply copy region for firstprivate. bool needsFirstprivate = @@ -1486,10 +1545,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, // in-place convert copy region builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator()); if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy", - builder, moduleTranslation))) { - bodyGenStatus = failure(); - return; - } + builder, moduleTranslation))) + return llvm::createStringError( + "failed to inline `copy` region of an `omp.private` op in the " + "parallel region"); // ignore unused value yielded from copy region @@ -1538,7 +1597,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, if (failed(inlineConvertOmpRegions( reductionDecls[i].getInitializerRegion(), "omp.reduction.neutral", builder, moduleTranslation, &phis))) - bodyGenStatus = failure(); + return llvm::createStringError( + "failed to inline `init` region of an `omp.declare_reduction` op " + "in the parallel region"); assert(phis.size() == 1 && "expected one value to be yielded from the " "reduction neutral element declaration region"); @@ -1582,9 +1643,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, // ParallelOp has only one region associated with it. builder.restoreIP(codeGenIP); - auto regionBlock = - convertOmpOpRegions(opInst.getRegion(), "omp.par.region", builder, - moduleTranslation, bodyGenStatus); + llvm::Expected regionBlock = convertOmpOpRegions( + opInst.getRegion(), "omp.par.region", builder, moduleTranslation); + if (!regionBlock) + return regionBlock.takeError(); // Process the reductions if required. if (opInst.getNumReductionVars() > 0) { @@ -1597,23 +1659,25 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, privateReductionVariables, reductionInfos); // Move to region cont block - builder.SetInsertPoint(regionBlock->getTerminator()); + builder.SetInsertPoint((*regionBlock)->getTerminator()); // Generate reductions from info llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable(); builder.SetInsertPoint(tempTerminator); - llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint = + llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint = ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos, isByRef, false); - if (!contInsertPoint.getBlock()) { - bodyGenStatus = opInst->emitOpError() << "failed to convert reductions"; - return; - } + if (!contInsertPoint) + return contInsertPoint.takeError(); + + if (!contInsertPoint->getBlock()) + return llvm::createStringError("failed to convert reductions"); tempTerminator->eraseFromParent(); - builder.restoreIP(contInsertPoint); + builder.restoreIP(*contInsertPoint); } + return llvm::Error::success(); }; auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP, @@ -1626,7 +1690,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) { + auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error { InsertPointTy oldIP = builder.saveIP(); builder.restoreIP(codeGenIP); @@ -1640,7 +1704,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, if (failed(inlineOmpRegionCleanup( reductionCleanupRegions, privateReductionVariables, moduleTranslation, builder, "omp.reduction.cleanup"))) - bodyGenStatus = failure(); + return llvm::createStringError( + "failed to inline `cleanup` region of an `omp.declare_reduction` op " + "in the parallel region"); SmallVector privateCleanupRegions; llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions), @@ -1651,9 +1717,11 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, if (failed(inlineOmpRegionCleanup( privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder, "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false))) - bodyGenStatus = failure(); + return llvm::createStringError("failed to inline `dealloc` region of an " + "`omp.private` op in the parallel region"); builder.restoreIP(oldIP); + return llvm::Error::success(); }; llvm::Value *ifCond = nullptr; @@ -1672,11 +1740,14 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP( + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB, - ifCond, numThreads, pbKind, isCancellable)); + ifCond, numThreads, pbKind, isCancellable); + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); - return bodyGenStatus; + builder.restoreIP(*afterIP); + return success(); } /// Convert Order attribute to llvm::omp::OrderKind. @@ -1718,12 +1789,10 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); // Generator of the canonical loop body. - // TODO: support error propagation in OpenMPIRBuilder and use it instead of - // relying on captured variables. SmallVector loopInfos; SmallVector bodyInsertPoints; - LogicalResult bodyGenStatus = success(); - auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) { + auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, + llvm::Value *iv) -> llvm::Error { // Make sure further conversions know about the induction variable. moduleTranslation.mapValue( loopOp.getRegion().front().getArgument(loopInfos.size()), iv); @@ -1734,12 +1803,13 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, bodyInsertPoints.push_back(ip); if (loopInfos.size() != loopOp.getNumLoops() - 1) - return; + return llvm::Error::success(); // Convert the body of the loop. builder.restoreIP(ip); - convertOmpOpRegions(loopOp.getRegion(), "omp.simd.region", builder, - moduleTranslation, bodyGenStatus); + return convertOmpOpRegions(loopOp.getRegion(), "omp.simd.region", builder, + moduleTranslation) + .takeError(); }; // Delegate actual loop construction to the OpenMP IRBuilder. @@ -1765,12 +1835,16 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, ompLoc.DL); computeIP = loopInfos.front()->getPreheaderIP(); } - loopInfos.push_back(ompBuilder->createCanonicalLoop( - loc, bodyGen, lowerBound, upperBound, step, - /*IsSigned=*/true, /*Inclusive=*/true, computeIP)); - if (failed(bodyGenStatus)) - return failure(); + llvm::Expected result = + ompBuilder->createCanonicalLoop( + loc, bodyGen, lowerBound, upperBound, step, + /*IsSigned=*/true, /*InclusiveStop=*/true, computeIP); + + if (!result) + return loopOp->emitError(llvm::toString(result.takeError())); + + loopInfos.push_back(*result); } // Collapse loops. @@ -1921,18 +1995,17 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, convertAtomicOrdering(opInst.getMemoryOrder()); // Generate update code. - LogicalResult updateGenStatus = success(); - auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus]( - llvm::Value *atomicx, - llvm::IRBuilder<> &builder) -> llvm::Value * { + auto updateFn = + [&opInst, &moduleTranslation]( + llvm::Value *atomicx, + llvm::IRBuilder<> &builder) -> llvm::Expected { Block &bb = *opInst.getRegion().begin(); moduleTranslation.mapValue(*opInst.getRegion().args_begin(), atomicx); moduleTranslation.mapBlock(&bb, builder.GetInsertBlock()); - if (failed(moduleTranslation.convertBlock(bb, true, builder))) { - updateGenStatus = (opInst.emitError() - << "unable to convert update operation to llvm IR"); - return nullptr; - } + if (failed(moduleTranslation.convertBlock(bb, true, builder))) + return llvm::createStringError( + "unable to convert update operation to llvm IR"); + omp::YieldOp yieldop = dyn_cast(bb.getTerminator()); assert(yieldop && yieldop.getResults().size() == 1 && "terminator must be omp.yield op and it must have exactly one " @@ -1943,10 +2016,16 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, // Handle ambiguous alloca, if any. auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(ompBuilder->createAtomicUpdate( - ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn, - isXBinopExpr)); - return updateGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr, + atomicOrdering, binop, updateFn, + isXBinopExpr); + + if (!afterIP) + return opInst.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } static LogicalResult @@ -2007,20 +2086,19 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, llvm::AtomicOrdering atomicOrdering = convertAtomicOrdering(atomicCaptureOp.getMemoryOrder()); - LogicalResult updateGenStatus = success(); - auto updateFn = [&](llvm::Value *atomicx, - llvm::IRBuilder<> &builder) -> llvm::Value * { + auto updateFn = + [&](llvm::Value *atomicx, + llvm::IRBuilder<> &builder) -> llvm::Expected { if (atomicWriteOp) return moduleTranslation.lookupValue(atomicWriteOp.getExpr()); Block &bb = *atomicUpdateOp.getRegion().begin(); moduleTranslation.mapValue(*atomicUpdateOp.getRegion().args_begin(), atomicx); moduleTranslation.mapBlock(&bb, builder.GetInsertBlock()); - if (failed(moduleTranslation.convertBlock(bb, true, builder))) { - updateGenStatus = (atomicUpdateOp.emitError() - << "unable to convert update operation to llvm IR"); - return nullptr; - } + if (failed(moduleTranslation.convertBlock(bb, true, builder))) + return llvm::createStringError( + "unable to convert update operation to llvm IR"); + omp::YieldOp yieldop = dyn_cast(bb.getTerminator()); assert(yieldop && yieldop.getResults().size() == 1 && "terminator must be omp.yield op and it must have exactly one " @@ -2031,10 +2109,16 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, // Handle ambiguous alloca, if any. auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation); llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); - builder.restoreIP(ompBuilder->createAtomicCapture( - ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering, - binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr)); - return updateGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = + ompBuilder->createAtomicCapture( + ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering, + binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr); + + if (!afterIP) + return atomicCaptureOp.emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } /// Converts an OpenMP Threadprivate operation into LLVM IR using @@ -3019,8 +3103,8 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, }; using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy; - LogicalResult bodyGenStatus = success(); - auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) { + auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { assert(isa(op) && "BodyGen requested for non TargetDataOp"); auto blockArgIface = cast(op); @@ -3046,8 +3130,10 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, return info.DevicePtrInfoMap[basePointer].second; }); - bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region", - builder, moduleTranslation); + if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder, + moduleTranslation))) + return llvm::createStringError( + "failed to inline region of an `omp.target_data` op"); } break; case BodyGenTy::DupNoPriv: @@ -3067,8 +3153,10 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, mapData.BasePointers, mapData.DevicePointers); } - bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region", - builder, moduleTranslation); + if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder, + moduleTranslation))) + return llvm::createStringError( + "failed to inline region of an `omp.target_data` op"); } break; } @@ -3078,17 +3166,21 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); llvm::OpenMPIRBuilder::InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation); - if (isa(op)) { - builder.restoreIP(ompBuilder->createTargetData( - ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond, - info, genMapInfoCB, nullptr, bodyGenCB)); - } else { - builder.restoreIP(ompBuilder->createTargetData( - ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond, - info, genMapInfoCB, &RTLFn)); - } - - return bodyGenStatus; + llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() { + if (isa(op)) + return ompBuilder->createTargetData( + ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), + ifCond, info, genMapInfoCB, nullptr, bodyGenCB); + return ompBuilder->createTargetData(ompLoc, allocaIP, builder.saveIP(), + builder.getInt64(deviceID), ifCond, + info, genMapInfoCB, &RTLFn); + }(); + + if (!afterIP) + return op->emitError(llvm::toString(afterIP.takeError())); + + builder.restoreIP(*afterIP); + return success(); } /// Lowers the FlagsAttr which is applied to the module on the device @@ -3320,6 +3412,8 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, } case omp::VariableCaptureKind::This: case omp::VariableCaptureKind::VLAType: + // TODO: Consider returning error to use standard reporting for + // unimplemented features. assert(false && "Currently unsupported capture kind"); break; } @@ -3350,10 +3444,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, bool isOffloadEntry = isTargetDevice || !ompBuilder->Config.TargetTriples.empty(); - LogicalResult bodyGenStatus = success(); using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - auto bodyCB = [&](InsertPointTy allocaIP, - InsertPointTy codeGenIP) -> InsertPointTy { + auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { // Forward target-cpu and target-features function attributes from the // original function to the new outlined function. llvm::Function *llvmParentFn = @@ -3396,34 +3489,35 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, if (privatizer.getDataSharingType() == omp::DataSharingClauseType::FirstPrivate || !privatizer.getDeallocRegion().empty()) { - opInst.emitError("Translation of omp.target from MLIR to LLVMIR " - "failed because translation of firstprivate and " - " private allocatables is not supported yet"); - bodyGenStatus = failure(); - } else { - moduleTranslation.mapValue(privatizer.getAllocMoldArg(), - moduleTranslation.lookupValue(privVar)); - Region &allocRegion = privatizer.getAllocRegion(); - SmallVector yieldedValues; - if (failed(inlineConvertOmpRegions( - allocRegion, "omp.targetop.privatizer", builder, - moduleTranslation, &yieldedValues))) { - opInst.emitError( - "failed to inline `alloc` region of an `omp.private` " - "op in the target region"); - bodyGenStatus = failure(); - } else { - assert(yieldedValues.size() == 1); - moduleTranslation.mapValue(privBlockArg, yieldedValues.front()); - } - moduleTranslation.forgetMapping(allocRegion); - builder.restoreIP(builder.saveIP()); + return llvm::createStringError( + "Translation of omp.target from MLIR to LLVMIR " + "failed because translation of firstprivate and " + " private allocatables is not supported yet"); } + moduleTranslation.mapValue(privatizer.getAllocMoldArg(), + moduleTranslation.lookupValue(privVar)); + Region &allocRegion = privatizer.getAllocRegion(); + SmallVector yieldedValues; + if (failed(inlineConvertOmpRegions( + allocRegion, "omp.targetop.privatizer", builder, + moduleTranslation, &yieldedValues))) { + return llvm::createStringError( + "failed to inline `alloc` region of an `omp.private` " + "op in the target region"); + } + assert(yieldedValues.size() == 1); + moduleTranslation.mapValue(privBlockArg, yieldedValues.front()); + moduleTranslation.forgetMapping(allocRegion); + builder.restoreIP(builder.saveIP()); } } - llvm::BasicBlock *exitBlock = convertOmpOpRegions( - targetRegion, "omp.target", builder, moduleTranslation, bodyGenStatus); - builder.SetInsertPoint(exitBlock); + + llvm::Expected exitBlock = convertOmpOpRegions( + targetRegion, "omp.target", builder, moduleTranslation); + if (!exitBlock) + return exitBlock.takeError(); + + builder.SetInsertPoint(*exitBlock); return builder.saveIP(); }; @@ -3455,7 +3549,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input, llvm::Value *&retVal, InsertPointTy allocaIP, - InsertPointTy codeGenIP) { + InsertPointTy codeGenIP) + -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { // We just return the unaltered argument for the host function // for now, some alterations may be required in the future to // keep host fallback functions working identically to the device @@ -3486,10 +3581,16 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, buildDependData(targetOp.getDependKinds(), targetOp.getDependVars(), moduleTranslation, dds); - builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget( - ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo, - defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB, - argAccessorCB, dds, targetOp.getNowait())); + llvm::OpenMPIRBuilder::InsertPointOrErrorTy result = + moduleTranslation.getOpenMPBuilder()->createTarget( + ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo, + defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB, + argAccessorCB, dds, targetOp.getNowait()); + + if (!result) + return opInst.emitError(llvm::toString(result.takeError())); + + builder.restoreIP(*result); // Remap access operations to declare target reference pointers for the // device, essentially generating extra loadop's as necessary @@ -3497,7 +3598,7 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, handleDeclareTargetMapVar(mapData, moduleTranslation, builder, llvmOutlinedFn); - return bodyGenStatus; + return success(); } static LogicalResult @@ -3618,8 +3719,13 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); return llvm::TypeSwitch(op) - .Case([&](omp::BarrierOp) { - ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier); + .Case([&](omp::BarrierOp) -> LogicalResult { + llvm::OpenMPIRBuilder::InsertPointOrErrorTy result = + ompBuilder->createBarrier(builder.saveIP(), + llvm::omp::OMPD_barrier); + if (!result) + return op->emitError(llvm::toString(result.takeError())); + return success(); }) .Case([&](omp::TaskyieldOp) {