overlap data transfer with application activities for SSC readers

ornladios · Jun 16, 2020 · ecf78fb · ecf78fb
1 parent ba786bb
commit ecf78fb
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 17 deletions.
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
@@ -56,14 +56,12 @@ void SscReader::GetOneSidedPostPush()
 {
     TAU_SCOPED_TIMER_FUNC();
     MPI_Win_post(m_MpiAllWritersGroup, 0, m_MpiWin);
-    MPI_Win_wait(m_MpiWin);
 }
 
 void SscReader::GetOneSidedFencePush()
 {
     TAU_SCOPED_TIMER_FUNC();
     MPI_Win_fence(0, m_MpiWin);
-    MPI_Win_fence(0, m_MpiWin);
 }
 
 void SscReader::GetOneSidedPostPull()
@@ -75,7 +73,6 @@ void SscReader::GetOneSidedPostPull()
         MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
                 i.first, 0, i.second.second, MPI_CHAR, m_MpiWin);
     }
-    MPI_Win_complete(m_MpiWin);
 }
 
 void SscReader::GetOneSidedFencePull()
@@ -87,21 +84,17 @@ void SscReader::GetOneSidedFencePull()
         MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
                 i.first, 0, i.second.second, MPI_CHAR, m_MpiWin);
     }
-    MPI_Win_fence(0, m_MpiWin);
 }
 
 void SscReader::GetTwoSided()
 {
     TAU_SCOPED_TIMER_FUNC();
-    std::vector<MPI_Request> requests;
     for (const auto &i : m_AllReceivingWriterRanks)
     {
-        requests.emplace_back();
+        m_MpiRequests.emplace_back();
         MPI_Irecv(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR,
-                  i.first, 0, m_StreamComm, &requests.back());
+                  i.first, 0, m_StreamComm, &m_MpiRequests.back());
     }
-    MPI_Status statuses[requests.size()];
-    MPI_Waitall(requests.size(), requests.data(), statuses);
 }
 
 StepStatus SscReader::BeginStep(const StepMode stepMode,
@@ -121,23 +114,25 @@ StepStatus SscReader::BeginStep(const StepMode stepMode,
         ++m_CurrentStep;
         if (m_MpiMode == "twosided")
         {
-            GetTwoSided();
+            MPI_Status statuses[m_MpiRequests.size()];
+            MPI_Waitall(m_MpiRequests.size(), m_MpiRequests.data(), statuses);
+            m_MpiRequests.clear();
         }
         else if (m_MpiMode == "onesidedfencepush")
         {
-            GetOneSidedFencePush();
+            MPI_Win_fence(0, m_MpiWin);
         }
         else if (m_MpiMode == "onesidedpostpush")
         {
-            GetOneSidedPostPush();
+            MPI_Win_wait(m_MpiWin);
         }
         else if (m_MpiMode == "onesidedfencepull")
         {
-            GetOneSidedFencePull();
+            MPI_Win_fence(0, m_MpiWin);
         }
         else if (m_MpiMode == "onesidedpostpull")
         {
-            GetOneSidedPostPull();
+            MPI_Win_complete(m_MpiWin);
         }
     }
 
@@ -227,10 +222,26 @@ void SscReader::EndStep()
         MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,
                        m_StreamComm, &m_MpiWin);
     }
-    if (m_Verbosity >= 5)
+
+    if (m_MpiMode == "twosided")
     {
-        std::cout << "SscReader::EndStep, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
+        GetTwoSided();
+    }
+    else if (m_MpiMode == "onesidedfencepush")
+    {
+        GetOneSidedFencePush();
+    }
+    else if (m_MpiMode == "onesidedpostpush")
+    {
+        GetOneSidedPostPush();
+    }
+    else if (m_MpiMode == "onesidedfencepull")
+    {
+        GetOneSidedFencePull();
+    }
+    else if (m_MpiMode == "onesidedpostpull")
+    {
+        GetOneSidedPostPull();
     }
 }
 

diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h
@@ -53,6 +53,7 @@ class SscReader : public Engine
     MPI_Group m_MpiAllWritersGroup;
     MPI_Comm m_StreamComm;
     std::string m_MpiMode = "twosided";
+    std::vector<MPI_Request> m_MpiRequests;
 
     int m_StreamRank;
     int m_StreamSize;