Merge pull request #1925 from DARMA-tasking/1909-cache-the-mpi-commun…

…icator-in-am 1909 Cache the MPI communicator in ActiveMessenger
DARMA-tasking · Aug 30, 2022 · ae0086d · ae0086d
2 parents 825ce69 + f1b75c6
commit ae0086d
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 7 deletions.
diff --git a/src/vt/context/context.h b/src/vt/context/context.h
@@ -221,7 +221,7 @@ struct Context : runtime::component::Component<Context> {
   NodeType thisNode_ = uninitialized_destination;
   NodeType numNodes_ = uninitialized_destination;
   WorkerCountType numWorkers_ = no_workers;
-  MPI_Comm communicator_ = MPI_COMM_WORLD;
+  MPI_Comm communicator_ = MPI_COMM_NULL;
   DeclareClassInsideInitTLS(Context, WorkerIDType, thisWorker_, no_worker_id)
   runnable::RunnableNew* cur_task_ = nullptr;
 };

diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc
@@ -152,6 +152,10 @@ ActiveMessenger::ActiveMessenger()
   };
 }
 
+void ActiveMessenger::initialize() {
+  comm_ = theContext()->getComm();
+}
+
 void ActiveMessenger::startup() {
   auto const this_node = theContext()->getNode();
   bare_handler_dummy_elm_id_for_lb_data_ =
@@ -370,7 +374,7 @@ EventType ActiveMessenger::sendMsgMPI(
       #endif
       int const ret = MPI_Isend(
         untyped_msg, small_msg_size, MPI_BYTE, dest, send_tag,
-        theContext()->getComm(), mpi_event->getRequest()
+        comm_, mpi_event->getRequest()
       );
       vtAssertMPISuccess(ret, "MPI_Isend");
 
@@ -606,7 +610,7 @@ std::tuple<EventType, int> ActiveMessenger::sendDataMPI(
 
       VT_ALLOW_MPI_CALLS;
       int const ret = MPI_Isend(
-        ptr, subsize, MPI_BYTE, dest, tag, theContext()->getComm(),
+        ptr, subsize, MPI_BYTE, dest, tag, comm_,
         mpi_event->getRequest()
       );
       vtAssertMPISuccess(ret, "MPI_Isend");
@@ -706,7 +710,7 @@ bool ActiveMessenger::recvDataMsgBuffer(
       VT_ALLOW_MPI_CALLS;
       const int probe_ret = MPI_Iprobe(
         node == uninitialized_destination ? MPI_ANY_SOURCE : node,
-        tag, theContext()->getComm(), &flag, &stat
+        tag, comm_, &flag, &stat
       );
       vtAssertMPISuccess(probe_ret, "MPI_Iprobe");
     }
@@ -791,7 +795,7 @@ void ActiveMessenger::recvDataDirect(
       VT_ALLOW_MPI_CALLS;
       int const ret = MPI_Irecv(
         cbuf+(i*max_per_send), sublen, MPI_BYTE, from, tag,
-        theContext()->getComm(), &reqs[i]
+        comm_, &reqs[i]
       );
       vtAssertMPISuccess(ret, "MPI_Irecv");
     }
@@ -1027,7 +1031,7 @@ bool ActiveMessenger::tryProcessIncomingActiveMsg() {
 
     MPI_Iprobe(
       MPI_ANY_SOURCE, static_cast<MPI_TagType>(MPITag::ActiveMsgTag),
-      theContext()->getComm(), &flag, &stat
+      comm_, &flag, &stat
     );
   }
 
@@ -1051,7 +1055,7 @@ bool ActiveMessenger::tryProcessIncomingActiveMsg() {
       VT_ALLOW_MPI_CALLS;
       MPI_Irecv(
         buf, num_probe_bytes, MPI_BYTE, sender, stat.MPI_TAG,
-        theContext()->getComm(), &req
+        comm_, &req
       );
 
       amPostedCounterGauge.incrementUpdate(num_probe_bytes, 1);

diff --git a/src/vt/messaging/active.h b/src/vt/messaging/active.h
@@ -341,6 +341,7 @@ struct ActiveMessenger : runtime::component::PollableComponent<ActiveMessenger>
   std::string name() override { return "ActiveMessenger"; }
 
   void startup() override;
+  void initialize() override;
 
   /**
    * \brief Mark a message as a termination message.
@@ -1785,6 +1786,7 @@ struct ActiveMessenger : runtime::component::PollableComponent<ActiveMessenger>
 private:
   elm::ElementIDStruct bare_handler_dummy_elm_id_for_lb_data_ = {};
   elm::ElementLBData bare_handler_lb_data_;
+  MPI_Comm comm_ = MPI_COMM_NULL;
 };
 
 }} // end namespace vt::messaging