Merge branch '708-vector-lb-stats' into develop

src/vt/vrt/collection/balance/elm_stats.impl.h

@@ -89,11 +89,12 @@ template <typename ColT>
   auto const& proxy = col->getCollectionProxy();
   auto const& untyped_proxy = col->getProxy();
   auto const& total_load = stats.getLoad(cur_phase, getFocusedSubPhase(untyped_proxy));
+  auto const& subphase_loads = stats.subphase_timings_.at(cur_phase);
   auto const& comm = stats.getComm(cur_phase);
   auto const& idx = col->getIndex();
   auto const& elm_proxy = proxy[idx];
 
-  theProcStats()->addProcStats<ColT>(elm_proxy, col, cur_phase, total_load, comm);
+  theProcStats()->addProcStats(col, cur_phase, total_load, subphase_loads, comm);
 
   auto const before_ready = theCollection()->numReadyCollections();
   theCollection()->makeCollectionReady(untyped_proxy);

src/vt/vrt/collection/balance/proc_stats.cc

@@ -109,6 +109,12 @@ ProcStats::getProcLoad(PhaseType phase) const {
   return proc_data_.at(phase);
 }
 
+ProcStats::SubphaseLoadMapType const&
+ProcStats::getProcSubphaseLoad(PhaseType phase) const {
+  vtAssert(proc_subphase_data_.size() > phase, "Phase must exist in load data");
+  return proc_subphase_data_.at(phase);
+}
+
 CommMapType const& ProcStats::getProcComm(PhaseType phase) const {
   vtAssert(proc_comm_.size() > phase, "Phase must exist in comm data");
   return proc_comm_.at(phase);
@@ -202,16 +208,29 @@ void ProcStats::outputStatsFile() {
 
   vtAssertExpr(stats_file_ != nullptr);
 
-  auto const num_iters = ProcStats::proc_data_.size();
+  auto const num_iters = proc_data_.size();
 
-  vt_print(lb, "outputStatsFile: file={}, iter={}\n", print_ptr(stats_file_), num_iters);
+  vt_print(lb, "ProcStats::outputStatsFile: file={}, iter={}\n", print_ptr(stats_file_), num_iters);
 
   for (size_t i = 0; i < num_iters; i++) {
-    for (auto&& elm : ProcStats::proc_data_.at(i)) {
-      auto obj_str = fmt::format("{},{},{}\n", i, elm.first, elm.second);
+    for (auto&& elm : proc_data_.at(i)) {
+      ElementIDType id = elm.first;
+      TimeType time = elm.second;
+      const auto& subphase_times = proc_subphase_data_.at(i)[id];
+      size_t subphases = subphase_times.size();
+
+      auto obj_str = fmt::format("{},{},{},{},[", i, id, time, subphases);
+      for (size_t s = 0; s < subphases; s++) {
+        obj_str += std::to_string(subphase_times[s]);
+        if (s != subphases - 1)
+          obj_str += ",";
+      }
+
+      obj_str += "]\n";
+
       fprintf(stats_file_, "%s", obj_str.c_str());
     }
-    for (auto&& elm : ProcStats::proc_comm_.at(i)) {
+    for (auto&& elm : proc_comm_.at(i)) {
       using E = typename std::underlying_type<CommCategory>::type;
 
       auto const& key = elm.first;
@@ -252,4 +271,59 @@ void ProcStats::outputStatsFile() {
   closeStatsFile();
 }
 
+ElementIDType ProcStats::addProcStats(
+  Migratable* col_elm,
+  PhaseType const& phase, TimeType const& time,
+  std::vector<TimeType> const& subphase_time, CommMapType const& comm
+) {
+  // A new temp ID gets assigned when a object is migrated into a node
+
+  auto const temp_id = col_elm->temp_elm_id_;
+  auto const perm_id = col_elm->stats_elm_id_;
+
+  debug_print(
+    lb, node,
+    "ProcStats::addProcStats: temp_id={}, perm_id={}, phase={}, subphases={}, load={}\n",
+    temp_id, perm_id, phase, subphase_time.size(), time
+  );
+
+  proc_data_.resize(phase + 1);
+  auto elm_iter = proc_data_.at(phase).find(temp_id);
+  vtAssert(elm_iter == proc_data_.at(phase).end(), "Must not exist");
+  proc_data_.at(phase).emplace(
+    std::piecewise_construct,
+    std::forward_as_tuple(temp_id),
+    std::forward_as_tuple(time)
+  );
+
+  proc_subphase_data_.resize(phase + 1);
+  auto elm_subphase_iter = proc_subphase_data_.at(phase).find(temp_id);
+  vtAssert(elm_subphase_iter == proc_subphase_data_.at(phase).end(), "Must not exist");
+  proc_subphase_data_.at(phase).emplace(
+    std::piecewise_construct,
+    std::forward_as_tuple(temp_id),
+    std::forward_as_tuple(subphase_time)
+  );
+
+  proc_comm_.resize(phase + 1);
+  for (auto&& c : comm) {
+    proc_comm_.at(phase)[c.first] += c.second;
+  }
+
+  proc_temp_to_perm_[temp_id] = perm_id;
+  proc_perm_to_temp_[perm_id] = temp_id;
+
+  auto migrate_iter = proc_migrate_.find(temp_id);
+  if (migrate_iter == proc_migrate_.end()) {
+    proc_migrate_.emplace(
+      std::piecewise_construct,
+      std::forward_as_tuple(temp_id),
+      std::forward_as_tuple([col_elm](NodeType node){
+        col_elm->migrate(node);
+      })
+    );
+  }
+  return temp_id;
+}
+
 }}}} /* end namespace vt::vrt::collection::balance */

src/vt/vrt/collection/balance/proc_stats.h

@@ -50,6 +50,7 @@
 #include "vt/vrt/collection/balance/lb_comm.h"
 #include "vt/vrt/collection/balance/phase_msg.h"
 #include "vt/vrt/collection/balance/stats_msg.h"
+#include "vt/vrt/collection/types/migratable.h"
 #include "vt/runtime/component/component_pack.h"
 #include "vt/timing/timing.h"
 #include "vt/objgroup/proxy/proxy_objgroup.h"
@@ -61,8 +62,9 @@
 namespace vt { namespace vrt { namespace collection { namespace balance {
 
 struct ProcStats : runtime::component::Component<ProcStats> {
-  using MigrateFnType = std::function<void(NodeType)>;
-  using LoadMapType   = std::unordered_map<ElementIDType,TimeType>;
+  using MigrateFnType       = std::function<void(NodeType)>;
+  using LoadMapType         = std::unordered_map<ElementIDType,TimeType>;
+  using SubphaseLoadMapType = std::unordered_map<ElementIDType, std::vector<TimeType>>;
 
   ProcStats() = default;
 
@@ -87,18 +89,17 @@ struct ProcStats : runtime::component::Component<ProcStats> {
   /**
    * \internal \brief Add processor statistics for local object
    *
-   * \param[in] elm_proxy the element proxy to the object
    * \param[in] col_elm the collection element pointer
    * \param[in] phase the current phase
    * \param[in] time the time the object took
    * \param[in] comm the comm graph for the object
    *
    * \return the temporary ID for the object assigned for this phase
    */
-  template <typename ColT>
   ElementIDType addProcStats(
-    VirtualElmProxyType<ColT> const& elm_proxy, ColT* col_elm,
-    PhaseType const& phase, TimeType const& time, CommMapType const& comm
+    Migratable* col_elm,
+    PhaseType const& phase, TimeType const& time,
+    std::vector<TimeType> const& subphase_time, CommMapType const& comm
   );
 
   /**
@@ -135,6 +136,15 @@ struct ProcStats : runtime::component::Component<ProcStats> {
    */
   LoadMapType const& getProcLoad(PhaseType phase) const;
 
+  /**
+   * \internal \brief Get object loads for the subphases of a given phase
+   *
+   * \param[in] phase the phase
+   *
+   * \return the subphase load map
+   */
+  SubphaseLoadMapType const& getProcSubphaseLoad(PhaseType phase) const;
+
   /**
    * \internal \brief Get object comm graph for a given phase
    *
@@ -198,6 +208,8 @@ struct ProcStats : runtime::component::Component<ProcStats> {
   objgroup::proxy::Proxy<ProcStats> proxy_;
   /// Processor timings for each local object
   std::vector<LoadMapType> proc_data_;
+  /// Processor subphase timings for each local object
+  std::vector<SubphaseLoadMapType> proc_subphase_data_;
   /// Local migration type-free lambdas for each object
   std::unordered_map<ElementIDType,MigrateFnType> proc_migrate_;
   /// Map of temporary ID to permanent ID
@@ -222,6 +234,4 @@ extern vrt::collection::balance::ProcStats* theProcStats();
 
 } /* end namespace vt */
 
-#include "vt/vrt/collection/balance/proc_stats.impl.h"
-
 #endif /*INCLUDED_VRT_COLLECTION_BALANCE_PROC_STATS_H*/

src/vt/vrt/collection/types/migratable.h

@@ -52,10 +52,14 @@
 #include "vt/vrt/collection/types/migratable.fwd.h"
 #include "vt/vrt/collection/balance/lb_common.h"
 #include "vt/vrt/collection/balance/elm_stats.h"
-#include "vt/vrt/collection/balance/proc_stats.h"
 
 namespace vt { namespace vrt { namespace collection {
 
+// Forward declaration for friend declaration below
+namespace balance {
+  struct ProcStats;
+}
+
 struct Migratable : MigrateHookBase {
 
   Migratable();