#1129: phase and elm: centralize subphases

DARMA-tasking · May 13, 2022 · be708b6 · be708b6
1 parent f5c9d8f
commit be708b6
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 19 deletions.
diff --git a/src/vt/elm/elm_lb_data.cc b/src/vt/elm/elm_lb_data.cc
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_ELM_ELM_LB_DATA_CC
 
 #include "vt/elm/elm_lb_data.h"
+#include "vt/phase/phase_manager.h"
 
 #include "vt/config.h"
 
@@ -87,17 +88,19 @@ void ElementLBData::sendToEntity(
 }
 
 void ElementLBData::sendComm(elm::CommKey key, double bytes) {
+  auto cur_subphase = thePhase()->getCurrentSubphase();
   phase_comm_[cur_phase_][key].sendMsg(bytes);
-  subphase_comm_[cur_phase_].resize(cur_subphase_ + 1);
-  subphase_comm_[cur_phase_].at(cur_subphase_)[key].sendMsg(bytes);
+  subphase_comm_[cur_phase_].resize(cur_subphase + 1);
+  subphase_comm_[cur_phase_].at(cur_subphase)[key].sendMsg(bytes);
 }
 
 void ElementLBData::recvComm(
   elm::CommKey key, double bytes
 ) {
+  auto cur_subphase = thePhase()->getCurrentSubphase();
   phase_comm_[cur_phase_][key].receiveMsg(bytes);
-  subphase_comm_[cur_phase_].resize(cur_subphase_ + 1);
-  subphase_comm_[cur_phase_].at(cur_subphase_)[key].receiveMsg(bytes);
+  subphase_comm_[cur_phase_].resize(cur_subphase + 1);
+  subphase_comm_[cur_phase_].at(cur_subphase)[key].receiveMsg(bytes);
 }
 
 void ElementLBData::recvObjData(
@@ -127,8 +130,9 @@ void ElementLBData::recvToNode(
 void ElementLBData::addTime(TimeTypeWrapper const& time) {
   phase_timings_[cur_phase_] += time.seconds();
 
-  subphase_timings_[cur_phase_].resize(cur_subphase_ + 1);
-  subphase_timings_[cur_phase_].at(cur_subphase_) += time.seconds();
+  auto cur_subphase = thePhase()->getCurrentSubphase();
+  subphase_timings_[cur_phase_].resize(cur_subphase + 1);
+  subphase_timings_[cur_phase_].at(cur_subphase) += time.seconds();
 
   vt_debug_print(
     verbose,lb,
@@ -199,6 +203,8 @@ TimeType ElementLBData::getLoad(PhaseType phase, SubphaseType subphase) const {
 }
 
 std::vector<TimeType> const& ElementLBData::getSubphaseTimes(PhaseType phase) {
+  auto cur_subphase = thePhase()->getCurrentSubphase();
+  subphase_timings_[phase].resize(cur_subphase + 1);
   return subphase_timings_[phase];
 }
 
@@ -216,6 +222,8 @@ ElementLBData::getComm(PhaseType const& phase) {
 }
 
 std::vector<CommMapType> const& ElementLBData::getSubphaseComm(PhaseType phase) {
+  auto cur_subphase = thePhase()->getCurrentSubphase();
+  subphase_comm_[phase].resize(cur_subphase + 1);
   auto const& subphase_comm = subphase_comm_[phase];
 
   vt_debug_print(
@@ -227,15 +235,6 @@ std::vector<CommMapType> const& ElementLBData::getSubphaseComm(PhaseType phase)
   return subphase_comm;
 }
 
-void ElementLBData::setSubPhase(SubphaseType subphase) {
-  vtAssert(subphase < no_subphase, "subphase must be less than sentinel");
-  cur_subphase_ = subphase;
-}
-
-SubphaseType ElementLBData::getSubPhase() const {
-  return cur_subphase_;
-}
-
 void ElementLBData::releaseLBDataFromUnneededPhases(PhaseType phase, unsigned int look_back) {
   if (phase >= look_back) {
     phase_timings_.erase(phase - look_back);

diff --git a/src/vt/elm/elm_lb_data.h b/src/vt/elm/elm_lb_data.h
@@ -90,8 +90,6 @@ struct ElementLBData {
   CommMapType const& getComm(PhaseType const& phase);
   std::vector<CommMapType> const& getSubphaseComm(PhaseType phase);
   std::vector<TimeType> const& getSubphaseTimes(PhaseType phase);
-  void setSubPhase(SubphaseType subphase);
-  SubphaseType getSubPhase() const;
 
   // these are just for unit testing
   std::size_t getLoadPhaseCount() const;
@@ -106,7 +104,6 @@ struct ElementLBData {
     s | cur_phase_;
     s | phase_timings_;
     s | phase_comm_;
-    s | cur_subphase_;
     s | subphase_timings_;
     s | subphase_comm_;
   }
@@ -129,7 +126,6 @@ struct ElementLBData {
   std::unordered_map<PhaseType, TimeType> phase_timings_ = {};
   std::unordered_map<PhaseType, CommMapType> phase_comm_ = {};
 
-  SubphaseType cur_subphase_ = 0;
   std::unordered_map<PhaseType, std::vector<TimeType>> subphase_timings_ = {};
   std::unordered_map<PhaseType, std::vector<CommMapType>> subphase_comm_ = {};
 };

diff --git a/src/vt/phase/phase_manager.cc b/src/vt/phase/phase_manager.cc
@@ -160,6 +160,7 @@ void PhaseManager::nextPhaseCollective() {
   runHooks(PhaseHook::EndPostMigration);
 
   cur_phase_++;
+  cur_subphase_ = 0;
 
   vt_debug_print(
     normal, phase,

diff --git a/src/vt/phase/phase_manager.h b/src/vt/phase/phase_manager.h
@@ -105,6 +105,18 @@ struct PhaseManager : runtime::component::Component<PhaseManager> {
    */
   PhaseType getCurrentPhase() const { return cur_phase_; }
 
+  /**
+   * \brief Get the current subphase
+   *
+   * \return the current subphase
+   */
+  SubphaseType getCurrentSubphase() const { return cur_subphase_; }
+
+  /**
+   * \brief Advance subphase
+   */
+  void advanceSubphase() { ++cur_subphase_; }
+
   /**
    * \brief Collectively register a phase hook that triggers depending on the
    * type of hook
@@ -200,6 +212,7 @@ struct PhaseManager : runtime::component::Component<PhaseManager> {
   template <typename SerializerT>
   void serialize(SerializerT& s) {
     s | cur_phase_
+      | cur_subphase_
       | proxy_
       | collective_hooks_
       | rooted_hooks_
@@ -213,6 +226,7 @@ struct PhaseManager : runtime::component::Component<PhaseManager> {
 
 private:
   PhaseType cur_phase_ = 0;                 /**< Current phase */
+  SubphaseType cur_subphase_ = 0;           /**< Current subphase */
   ObjGroupProxyType proxy_ = no_obj_group;  /**< Objgroup proxy  */
   HookIDMapType collective_hooks_;          /**< Collective regisstered hooks */
   HookIDMapType rooted_hooks_;              /**< Rooted regisstered hooks  */