#1909: Cache the communicator

DARMA-tasking · Aug 30, 2022 · 287fd57 · 287fd57
1 parent 825ce69
commit 287fd57
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 7 deletions.
diff --git a/src/vt/context/context.h b/src/vt/context/context.h
@@ -221,7 +221,7 @@ struct Context : runtime::component::Component<Context> {
   NodeType thisNode_ = uninitialized_destination;
   NodeType numNodes_ = uninitialized_destination;
   WorkerCountType numWorkers_ = no_workers;
-  MPI_Comm communicator_ = MPI_COMM_WORLD;
+  MPI_Comm communicator_ = MPI_COMM_NULL;
   DeclareClassInsideInitTLS(Context, WorkerIDType, thisWorker_, no_worker_id)
   runnable::RunnableNew* cur_task_ = nullptr;
 };

diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc
@@ -157,6 +157,8 @@ void ActiveMessenger::startup() {
   bare_handler_dummy_elm_id_for_lb_data_ =
     elm::ElmIDBits::createBareHandler(this_node);
 
+  comm_ = theContext()->getComm();
+
 #if vt_check_enabled(lblite)
   // Hook to collect LB data about objgroups
   thePhase()->registerHookCollective(phase::PhaseHook::DataCollection, [this]{
@@ -370,7 +372,7 @@ EventType ActiveMessenger::sendMsgMPI(
       #endif
       int const ret = MPI_Isend(
         untyped_msg, small_msg_size, MPI_BYTE, dest, send_tag,
-        theContext()->getComm(), mpi_event->getRequest()
+        comm_, mpi_event->getRequest()
       );
       vtAssertMPISuccess(ret, "MPI_Isend");
 
@@ -606,7 +608,7 @@ std::tuple<EventType, int> ActiveMessenger::sendDataMPI(
 
       VT_ALLOW_MPI_CALLS;
       int const ret = MPI_Isend(
-        ptr, subsize, MPI_BYTE, dest, tag, theContext()->getComm(),
+        ptr, subsize, MPI_BYTE, dest, tag, comm_,
         mpi_event->getRequest()
       );
       vtAssertMPISuccess(ret, "MPI_Isend");
@@ -706,7 +708,7 @@ bool ActiveMessenger::recvDataMsgBuffer(
       VT_ALLOW_MPI_CALLS;
       const int probe_ret = MPI_Iprobe(
         node == uninitialized_destination ? MPI_ANY_SOURCE : node,
-        tag, theContext()->getComm(), &flag, &stat
+        tag, comm_, &flag, &stat
       );
       vtAssertMPISuccess(probe_ret, "MPI_Iprobe");
     }
@@ -791,7 +793,7 @@ void ActiveMessenger::recvDataDirect(
       VT_ALLOW_MPI_CALLS;
       int const ret = MPI_Irecv(
         cbuf+(i*max_per_send), sublen, MPI_BYTE, from, tag,
-        theContext()->getComm(), &reqs[i]
+        comm_, &reqs[i]
       );
       vtAssertMPISuccess(ret, "MPI_Irecv");
     }
@@ -1027,7 +1029,7 @@ bool ActiveMessenger::tryProcessIncomingActiveMsg() {
 
     MPI_Iprobe(
       MPI_ANY_SOURCE, static_cast<MPI_TagType>(MPITag::ActiveMsgTag),
-      theContext()->getComm(), &flag, &stat
+      comm_, &flag, &stat
     );
   }
 
@@ -1051,7 +1053,7 @@ bool ActiveMessenger::tryProcessIncomingActiveMsg() {
       VT_ALLOW_MPI_CALLS;
       MPI_Irecv(
         buf, num_probe_bytes, MPI_BYTE, sender, stat.MPI_TAG,
-        theContext()->getComm(), &req
+        comm_, &req
       );
 
       amPostedCounterGauge.incrementUpdate(num_probe_bytes, 1);

diff --git a/src/vt/messaging/active.h b/src/vt/messaging/active.h
@@ -1785,6 +1785,7 @@ struct ActiveMessenger : runtime::component::PollableComponent<ActiveMessenger>
 private:
   elm::ElementIDStruct bare_handler_dummy_elm_id_for_lb_data_ = {};
   elm::ElementLBData bare_handler_lb_data_;
+  MPI_Comm comm_ = MPI_COMM_NULL;
 };
 
 }} // end namespace vt::messaging