From 1cbbd80c5a969f87f4b2b07c8055fe10dcdc282c Mon Sep 17 00:00:00 2001
From: Jakub Strzebonski <jakub.strzebonski@ng-analytics.com>
Date: Thu, 10 Sep 2020 21:38:17 +0200
Subject: [PATCH] #1014 revert pointless changes (squash this before merge)

---
 .../balance/lb_invoke/lb_manager.cc           |  8 ----
 src/vt/vrt/collection/balance/node_stats.cc   | 43 ++++++++++++++++++-
 src/vt/vrt/collection/balance/node_stats.h    |  4 ++
 src/vt/vrt/collection/manager.cc              | 18 --------
 src/vt/vrt/collection/manager.h               |  1 -
 5 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc
index acd21bfc58..4b30c27d90 100644
--- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc
+++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc
@@ -178,14 +178,6 @@ LBManager::makeLB(MsgSharedPtr<StartLBMsg> msg) {
       lb, node,
       "LBManager: finished migrations\n"
     );
-
-  // Statistics output when LB is enabled and appropriate flag is enabled
-#if vt_check_enabled(lblite)
-  if (theConfig()->vt_lb_stats) {
-    theNodeStats()->outputStatsForPhase(phase);
-  }
-#endif
-
     theNodeStats()->startIterCleanup(phase, model_->getNumPastPhasesNeeded());
     this->finishedRunningLB(phase);
   });
diff --git a/src/vt/vrt/collection/balance/node_stats.cc b/src/vt/vrt/collection/balance/node_stats.cc
index db34eee295..aa912e325a 100644
--- a/src/vt/vrt/collection/balance/node_stats.cc
+++ b/src/vt/vrt/collection/balance/node_stats.cc
@@ -142,6 +142,13 @@ void NodeStats::startIterCleanup(PhaseType phase, int look_back) {
   }
   node_data_[phase] = std::move(new_data);
 
+  // Statistics output when LB is enabled and appropriate flag is enabled
+#if vt_check_enabled(lblite)
+  if (theConfig()->vt_lb_stats) {
+    outputStatsForPhase(phase);
+  }
+#endif
+
   if (phase - look_back >= 0) {
     node_data_.erase(phase - look_back);
     node_subphase_data_.erase(phase - look_back);
@@ -170,6 +177,14 @@ void NodeStats::releaseLB() {
   CollectionManager::releaseLBPhase(msg_hold.get());
 }
 
+void NodeStats::initialize() {
+#if vt_check_enabled(lblite)
+  if (theConfig()->vt_lb_stats) {
+    theNodeStats()->createStatsFile();
+  }
+#endif
+}
+
 void NodeStats::createStatsFile() {
   auto const node = theContext()->getNode();
   auto const base_file = std::string(theConfig()->vt_lb_stats_file);
@@ -182,7 +197,33 @@ void NodeStats::createStatsFile() {
     "NodeStats: createStatsFile file={}\n", file_name
   );
 
+  // Node 0 creates the directory
+  if (not created_dir_ and node == 0) {
+    mkdir(dir.c_str(), S_IRWXU);
+    created_dir_ = true;
+  }
+
+  // Barrier: wait for node 0 to create directory before trying to put a file in
+  // the stats destination directory
+  if (curRT) {
+    curRT->systemSync();
+  } else {
+    // Something is wrong
+    vtAssert(false, "Trying to dump stats when VT runtime is deallocated?");
+  }
+
   stats_file_ = fopen(file_name.c_str(), "w+");
+  vtAssertExpr(stats_file_ != nullptr);
+}
+
+void NodeStats::finalize() {
+  // If statistics are enabled, close output file and clear stats
+#if vt_check_enabled(lblite)
+  if (theConfig()->vt_lb_stats) {
+    closeStatsFile();
+    clearStats();
+  }
+#endif
 }
 
 void NodeStats::closeStatsFile() {
@@ -195,7 +236,7 @@ void NodeStats::closeStatsFile() {
 void NodeStats::outputStatsForPhase(PhaseType phase) {
   vtAssertExpr(stats_file_ != nullptr);
 
-  vt_print(lb, "NodeStats::outputStatsFile: file={}, phase={}\n", print_ptr(stats_file_), phase);
+  vt_print(lb, "NodeStats::outputStatsForPhase: file={}, phase={}\n", print_ptr(stats_file_), phase);
 
   auto i = phase;
   for (auto&& elm : node_data_.at(i)) {
diff --git a/src/vt/vrt/collection/balance/node_stats.h b/src/vt/vrt/collection/balance/node_stats.h
index 355a1de26e..24d3b9846a 100644
--- a/src/vt/vrt/collection/balance/node_stats.h
+++ b/src/vt/vrt/collection/balance/node_stats.h
@@ -237,6 +237,10 @@ struct NodeStats : runtime::component::Component<NodeStats> {
    */
   VirtualProxyType getCollectionProxyForElement(ElementIDType temp_id) const;
 
+  void initialize() override;
+  void finalize() override;
+
+private:
   /**
    * \internal \brief Create the stats file
    */
diff --git a/src/vt/vrt/collection/manager.cc b/src/vt/vrt/collection/manager.cc
index 4bc2b1b7ec..0d15168387 100644
--- a/src/vt/vrt/collection/manager.cc
+++ b/src/vt/vrt/collection/manager.cc
@@ -54,26 +54,8 @@ namespace vt { namespace vrt { namespace collection {
 
 CollectionManager::CollectionManager() { }
 
-void CollectionManager::initialize() {
-  // If statistics are enabled create output directory and file
-#if vt_check_enabled(lblite)
-  if (theConfig()->vt_lb_stats) {
-    mkdir(theConfig()->vt_lb_stats_dir.c_str(), S_IRWXU);
-    theNodeStats()->createStatsFile();
-  }
-#endif
-}
-
 void CollectionManager::finalize() {
   cleanupAll<>();
-
-  // If statistics are enabled, close output file and clear stats
-#if vt_check_enabled(lblite)
-  if (theConfig()->vt_lb_stats) {
-    theNodeStats()->closeStatsFile();
-    theNodeStats()->clearStats();
-  }
-#endif
 }
 
 /*virtual*/ CollectionManager::~CollectionManager() { }
diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h
index 430bb00ed9..2872ba411d 100644
--- a/src/vt/vrt/collection/manager.h
+++ b/src/vt/vrt/collection/manager.h
@@ -157,7 +157,6 @@ struct CollectionManager
 
   virtual ~CollectionManager();
 
-  void initialize() override;
   void finalize() override;
 
   std::string name() override { return "CollectionManager"; }