diff --git a/cmake/link_vt.cmake b/cmake/link_vt.cmake index 8b6303404b..1114cf7fbd 100644 --- a/cmake/link_vt.cmake +++ b/cmake/link_vt.cmake @@ -236,4 +236,11 @@ function(link_target_with_vt) if (vt_ubsan_enabled) target_link_libraries(${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} -fsanitize=undefined) endif() + + # Enable additional flag for GCC-8 to link std::filesystem + if (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU") + if (NOT (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9)) + target_link_libraries(${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} -lstdc++fs) + endif () + endif () endfunction() diff --git a/src/vt/collective/collective_ops.cc b/src/vt/collective/collective_ops.cc index b69d7d1122..022b457de8 100644 --- a/src/vt/collective/collective_ops.cc +++ b/src/vt/collective/collective_ops.cc @@ -293,8 +293,10 @@ void CollectiveAnyOps::abort( auto myrt = tls_rt ? tls_rt : ::vt::rt; if (myrt) { #if vt_check_enabled(trace_enabled) - //--- Try to flush most of the traces before aborting - myrt->theTrace->cleanupTracesFile(); + if (myrt->theTrace) { + //--- Try to flush most of the traces before aborting + myrt->theTrace->cleanupTracesFile(); + } #endif myrt->abort(str, code); } else if (vt::debug::preConfig()->vt_throw_on_abort) { diff --git a/src/vt/configs/error/stack_out.cc b/src/vt/configs/error/stack_out.cc index 3dd30029bd..97c5fd1520 100644 --- a/src/vt/configs/error/stack_out.cc +++ b/src/vt/configs/error/stack_out.cc @@ -164,7 +164,7 @@ std::string prettyPrintStack(DumpStackType const& stack) { auto magenta = ::vt::debug::magenta(); auto yellow = ::vt::debug::yellow(); auto vt_pre = ::vt::debug::vtPre(); - auto node = ::vt::theContext()->getNode(); + auto node = ::vt::theContext() ? ::vt::theContext()->getNode() : -1; auto node_str = ::vt::debug::proc(node); auto prefix = vt_pre + node_str + " "; auto separator = fmt::format("{}{}{:-^120}{}\n", prefix, yellow, "", reset); diff --git a/src/vt/runtime/runtime.cc b/src/vt/runtime/runtime.cc index f8ad7f36d1..d87182cd37 100644 --- a/src/vt/runtime/runtime.cc +++ b/src/vt/runtime/runtime.cc @@ -424,12 +424,25 @@ bool Runtime::tryFinalize(bool const disable_sig) { } bool Runtime::needLBDataRestartReader() { + using vrt::collection::balance::ReadLBConfig; + using vrt::collection::balance::LBType; + using vrt::collection::balance::get_lb_names; + + bool needOfflineLB = false; + #if vt_check_enabled(lblite) - if (true) { - return arg_config_->config_.vt_lb_data_in; - } else + if (ReadLBConfig::openConfig(arg_config_->config_.vt_lb_file_name)) { + needOfflineLB = ReadLBConfig::hasOfflineLB(); + } + + needOfflineLB = needOfflineLB || arg_config_->config_.vt_lb_name == get_lb_names()[LBType::OfflineLB]; + + if (needOfflineLB && !arg_config_->config_.vt_lb_data_in) { + vtAbort("VT cannot run OfflineLB without '--vt_lb_data_in' parameter."); + } #endif - return false; + + return needOfflineLB; } bool Runtime::initialize(bool const force_now) { @@ -566,7 +579,7 @@ void Runtime::reset() { void Runtime::abort(std::string const abort_str, ErrorCodeType const code) { output(abort_str, code, true, true, false); - if (theConfig()->vt_throw_on_abort) { + if (theContext && theConfig()->vt_throw_on_abort) { throw std::runtime_error(abort_str); } else { aborted_ = true; @@ -627,7 +640,12 @@ void Runtime::output( fmt::print(stderr, "{}\n", prefix); } - if (!theConfig()->vt_no_stack) { + if (theContext == nullptr) { + // Too early in init process to check dump settings - always dump stack. + auto stack = debug::stack::dumpStack(); + auto stack_pretty = debug::stack::prettyPrintStack(stack); + fmt::print("{}", stack_pretty); + } else if (!theConfig()->vt_no_stack) { bool const on_abort = !theConfig()->vt_no_abort_stack; bool const on_warn = !theConfig()->vt_no_warn_stack; bool const dump = (error && on_abort) || (!error && on_warn); diff --git a/src/vt/vrt/collection/balance/lb_data_restart_reader.cc b/src/vt/vrt/collection/balance/lb_data_restart_reader.cc index 91c763a30f..2fe43be8f3 100644 --- a/src/vt/vrt/collection/balance/lb_data_restart_reader.cc +++ b/src/vt/vrt/collection/balance/lb_data_restart_reader.cc @@ -79,19 +79,45 @@ void LBDataRestartReader::startup() { } void LBDataRestartReader::readHistory(LBDataHolder const& lbdh) { - num_phases_ = lbdh.node_data_.size(); + auto find_max_data_phase = [&]() -> PhaseType { + if (lbdh.node_data_.empty()) { + return 0; + } + return std::max_element( + lbdh.node_data_.begin(), lbdh.node_data_.end(), + [](const auto& p1, const auto& p2) { return p1.first < p2.first; }) + ->first; + }; + + // Find last phase number + auto largest_data = find_max_data_phase(); + auto largest_identical = + lbdh.identical_phases_.size() > 0 ? *lbdh.identical_phases_.rbegin() : 0; + auto largest_skipped = + lbdh.skipped_phases_.size() > 0 ? *lbdh.skipped_phases_.rbegin() : 0; + num_phases_ = + std::max(std::max(largest_data, largest_identical), largest_skipped) + 1; + + PhaseType last_found_phase = 0; for (PhaseType phase = 0; phase < num_phases_; phase++) { auto iter = lbdh.node_data_.find(phase); if (iter != lbdh.node_data_.end()) { + last_found_phase = phase; for (auto const& obj : iter->second) { if (obj.first.isMigratable()) { - history_[phase].insert(obj.first); + if (history_[phase] == nullptr) { + history_[phase] = std::make_shared>(); + } + history_[phase]->insert(obj.first); } } - } else { - // We assume that all phases are dense all fully specified even if they - // don't change - vtAbort("Could not find data: phases must all be specified"); + } else if ( + lbdh.identical_phases_.find(phase) != lbdh.identical_phases_.end()) { + // Phase is identical to previous one, use the shared pointer to data from previous phase + addIdenticalPhase(phase, last_found_phase); + } else if (lbdh.skipped_phases_.find(phase) == lbdh.skipped_phases_.end()) { + vtAbort("Could not find data: Skipped phases needs to be listed in file " + "metadata."); } } } @@ -134,12 +160,12 @@ void LBDataRestartReader::arriving(ArriveMsg* msg) { } void LBDataRestartReader::update(UpdateMsg* msg) { - auto iter = history_[msg->phase].find(msg->elm); - vtAssert(iter != history_[msg->phase].end(), "Must exist"); + auto iter = history_[msg->phase]->find(msg->elm); + vtAssert(iter != history_[msg->phase]->end(), "Must exist"); auto elm = *iter; elm.curr_node = msg->curr_node; - history_[msg->phase].erase(iter); - history_[msg->phase].insert(elm); + history_[msg->phase]->erase(iter); + history_[msg->phase]->insert(elm); } void LBDataRestartReader::checkBothEnds(Coord& coord) { @@ -155,30 +181,31 @@ void LBDataRestartReader::determinePhasesToMigrate() { local_changed_distro.resize(num_phases_ - 1); auto const this_node = theContext()->getNode(); - runInEpochCollective("LBDataRestartReader::updateLocations", [&]{ for (PhaseType i = 0; i < num_phases_ - 1; ++i) { - local_changed_distro[i] = history_[i] != history_[i+1]; - if (local_changed_distro[i]) { - std::set departing, arriving; - - std::set_difference( - history_[i+1].begin(), history_[i+1].end(), - history_[i].begin(), history_[i].end(), - std::inserter(arriving, arriving.begin()) - ); - - std::set_difference( - history_[i].begin(), history_[i].end(), - history_[i+1].begin(), history_[i+1].end(), - std::inserter(departing, departing.begin()) - ); - - for (auto&& d : departing) { - proxy_[d.getHomeNode()].send(this_node, i+1, d); - } - for (auto&& a : arriving) { - proxy_[a.getHomeNode()].send(this_node, i+1, a); + if(history_.count(i) && history_.count(i+1)) { + local_changed_distro[i] = *history_[i] != *history_[i+1]; + if (local_changed_distro[i]) { + std::set departing, arriving; + + std::set_difference( + history_[i+1]->begin(), history_[i+1]->end(), + history_[i]->begin(), history_[i]->end(), + std::inserter(arriving, arriving.begin()) + ); + + std::set_difference( + history_[i]->begin(), history_[i]->end(), + history_[i+1]->begin(), history_[i+1]->end(), + std::inserter(departing, departing.begin()) + ); + + for (auto&& d : departing) { + proxy_[d.getHomeNode()].send(this_node, i+1, d); + } + for (auto&& a : arriving) { + proxy_[a.getHomeNode()].send(this_node, i+1, a); + } } } } diff --git a/src/vt/vrt/collection/balance/lb_data_restart_reader.h b/src/vt/vrt/collection/balance/lb_data_restart_reader.h index 0b90e014a1..6373a81153 100644 --- a/src/vt/vrt/collection/balance/lb_data_restart_reader.h +++ b/src/vt/vrt/collection/balance/lb_data_restart_reader.h @@ -128,11 +128,11 @@ struct LBDataRestartReader : runtime::component::Component * * \param[in] phase the phase * - * \return element assigned to this node + * \return pointer to elements assigned to this node if not skipped */ - std::set const& getDistro(PhaseType phase) { + std::shared_ptr> getDistro(PhaseType phase) const { auto iter = history_.find(phase); - vtAssert(iter != history_.end(), "Must have a valid phase"); + vtAssert(iter != history_.end(), "Must have a valid, not skipped phase"); return iter->second; } @@ -142,10 +142,31 @@ struct LBDataRestartReader : runtime::component::Component * \param[in] phase the phase to clear */ void clearDistro(PhaseType phase) { - auto iter = history_.find(phase); - if (iter != history_.end()) { - history_.erase(iter); + history_.erase(phase); + } + + /** + * \brief Add history for a given phase + * + * \param[in] phase the phase to be added + * \param[in] distro the distribution to be added + */ + void addDistro(PhaseType phase, const std::set& distro) { + if (history_[phase] == nullptr) { + history_[phase] = std::make_shared>(); } + history_[phase]->insert(distro.begin(), distro.end()); + } + + /** + * \brief Add identical phase to one already present + * + * \param[in] phase the phase to be added + * \param[in] identical the identical phase to be used + */ + void addIdenticalPhase(PhaseType phase, PhaseType identical) { + vtAssert(history_.find(identical) != history_.end(), "Identical phase was not added to history map."); + history_[phase] = history_[identical]; } private: @@ -170,7 +191,7 @@ struct LBDataRestartReader : runtime::component::Component std::vector changed_distro_; /// History of mapping that was read in from the data files - std::unordered_map> history_; + std::unordered_map>> history_; struct DepartMsg : vt::Message { DepartMsg(NodeType in_depart_node, PhaseType in_phase, ElementIDStruct in_elm) diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc index 2de4a94fae..873632ae90 100644 --- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc +++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc @@ -150,6 +150,11 @@ LBType LBManager::decideLBToRun(PhaseType phase, bool try_file) { } } + // Check if LBDataRestartReader requires to run OfflineLB for a given phase. + if(the_lb == LBType::OfflineLB && !theLBDataReader()->needsLB(phase)) { + the_lb = LBType::NoLB; + } + vt_debug_print( terse, lb, "LBManager::decidedLBToRun: phase={}, return lb_={}\n", diff --git a/src/vt/vrt/collection/balance/offlinelb/offlinelb.cc b/src/vt/vrt/collection/balance/offlinelb/offlinelb.cc index 0ed7636aab..b9abd6d03f 100644 --- a/src/vt/vrt/collection/balance/offlinelb/offlinelb.cc +++ b/src/vt/vrt/collection/balance/offlinelb/offlinelb.cc @@ -55,8 +55,8 @@ void OfflineLB::init(objgroup::proxy::Proxy in_proxy) { } void OfflineLB::runLB(LoadType) { - auto const& distro = theLBDataReader()->getDistro(phase_ + 1); - for (auto&& elm : distro) { + auto const distro = theLBDataReader()->getDistro(phase_ + 1); + for (auto&& elm : *distro) { migrateObjectTo(elm, theContext()->getNode()); } theLBDataReader()->clearDistro(phase_ + 1); diff --git a/src/vt/vrt/collection/balance/read_lb.cc b/src/vt/vrt/collection/balance/read_lb.cc index f2aa22d20d..d85edf680a 100644 --- a/src/vt/vrt/collection/balance/read_lb.cc +++ b/src/vt/vrt/collection/balance/read_lb.cc @@ -61,6 +61,7 @@ namespace vt { namespace vrt { namespace collection { namespace balance { /*static*/ typename ReadLBConfig::ConfigMapType ReadLBConfig::config_exact_ = {}; /*static*/ std::vector ReadLBConfig::config_prec_ = {}; /*static*/ bool ReadLBConfig::read_complete_ = false; +/*static*/ bool ReadLBConfig::has_offline_lb_ = false; /*static*/ bool ReadLBConfig::openConfig(std::string const& filename) { // No-op if no file specified. Can't be used to clear. @@ -231,6 +232,10 @@ int eatWhitespace(std::ifstream& file) { vtAbort(err_msg); } + if (lb_name == get_lb_names()[LBType::OfflineLB]) { + has_offline_lb_ = true; + } + map->emplace( std::piecewise_construct, std::forward_as_tuple(mod), @@ -243,6 +248,7 @@ int eatWhitespace(std::ifstream& file) { /*static*/ void ReadLBConfig::clear() { read_complete_ = false; + has_offline_lb_ = false; open_filename_ = ""; config_mod_.clear(); config_exact_.clear(); diff --git a/src/vt/vrt/collection/balance/read_lb.h b/src/vt/vrt/collection/balance/read_lb.h index a1fbb6998a..03d409307f 100644 --- a/src/vt/vrt/collection/balance/read_lb.h +++ b/src/vt/vrt/collection/balance/read_lb.h @@ -197,6 +197,7 @@ struct ReadLBConfig { static ConfigIndex numEntries() { return config_mod_.size() + config_exact_.size(); } static ConfigEntry* entry(ConfigIndex const& idx); static LBType getLB(ConfigIndex const& idx); + static bool hasOfflineLB() { return has_offline_lb_; }; static ConfigMapType getModEntries() { return config_mod_; }; static ConfigMapType getExactEntries() {return config_exact_; }; static ParamMapType parseParams(std::vector params); @@ -208,6 +209,7 @@ struct ReadLBConfig { static void readFile(std::string const& filename); static bool read_complete_; + static bool has_offline_lb_; static std::string open_filename_; static ConfigMapType config_mod_; static ConfigMapType config_exact_; diff --git a/tests/unit/lb/test_lb_reader.nompi.cc b/tests/unit/lb/test_lb_reader.nompi.cc index 3451bab7a7..5da78769d9 100644 --- a/tests/unit/lb/test_lb_reader.nompi.cc +++ b/tests/unit/lb/test_lb_reader.nompi.cc @@ -44,6 +44,7 @@ #include #include "test_harness.h" +#include "test_helpers.h" namespace vt { namespace tests { namespace unit { @@ -51,7 +52,7 @@ using TestLBReader = TestHarness; TEST_F(TestLBReader, test_lb_read_1) { - std::string file_name = "test_lb_read_1.txt"; + std::string file_name = getUniqueFilename(".txt"); std::ofstream out(file_name); out << "" "0 NoLB\n" @@ -66,6 +67,7 @@ TEST_F(TestLBReader, test_lb_read_1) { Config::clear(); Config::openConfig(file_name); + EXPECT_EQ(Config::hasOfflineLB(), false); EXPECT_EQ(Config::numEntries(), 3); EXPECT_EQ(Config::getExactEntries().size(), 2); EXPECT_EQ(Config::getModEntries().size(), 1); @@ -104,7 +106,7 @@ TEST_F(TestLBReader, test_lb_read_1) { TEST_F(TestLBReader, test_lb_read_2) { - std::string file_name = "test_lb_read_2.txt"; + std::string file_name = getUniqueFilename(".txt"); std::ofstream out(file_name); out << "" "0 NoLB\n" @@ -120,6 +122,7 @@ TEST_F(TestLBReader, test_lb_read_2) { Config::clear(); Config::openConfig(file_name); + EXPECT_EQ(Config::hasOfflineLB(), false); EXPECT_EQ(Config::numEntries(), 5); for (ConfigIdx i = 0; i < 121; i++) { auto entry = Config::entry(i); @@ -195,4 +198,20 @@ TEST_F(TestLBReader, test_lb_read_2) { EXPECT_EQ(Config::toString(), expected_config); } +TEST_F(TestLBReader, test_lb_read_3_with_offline_lb) { + std::string file_name = getUniqueFilename(".txt"); + std::ofstream out(file_name); + out << "" + "0 NoLB\n" + "1 OfflineLB\n" + "%10 OfflineLB\n"; + out.close(); + + using Config = vt::vrt::collection::balance::ReadLBConfig; + + Config::clear(); + Config::openConfig(file_name); + EXPECT_EQ(Config::hasOfflineLB(), true); +} + }}} // end namespace vt::tests::unit diff --git a/tests/unit/lb/test_offlinelb.cc b/tests/unit/lb/test_offlinelb.cc index 981e09d801..541b9225e7 100644 --- a/tests/unit/lb/test_offlinelb.cc +++ b/tests/unit/lb/test_offlinelb.cc @@ -48,6 +48,7 @@ #include #include "test_parallel_harness.h" +#include "test_helpers.h" namespace vt { namespace tests { namespace unit { namespace lb { @@ -74,6 +75,21 @@ struct SimCol : vt::Collection { EXPECT_EQ(getIndex().x() / 2, next_node); } } + + void sparseHandler(Msg* m){ + auto const this_node = theContext()->getNode(); + auto const num_nodes = theContext()->getNumNodes(); + auto const next_node = (this_node + 1) % num_nodes; + auto const prev_node = this_node - 1 >= 0 ? this_node - 1 : num_nodes - 1; + vt_debug_print(terse, lb, "sparseHandler: idx={}: elm={}\n", getIndex(), getElmID()); + if (m->iter <= 3 or m->iter == 6) { + EXPECT_EQ(getIndex().x() / 2, this_node); + } else if (m->iter == 4 or m-> iter == 5) { + EXPECT_EQ(getIndex().x() / 2, prev_node); + } else if (m->iter == 7 or m->iter == 8 or m->iter == 9) { + EXPECT_EQ(getIndex().x() / 2, next_node); + } + } }; TEST_F(TestOfflineLB, test_offlinelb_1) { @@ -149,6 +165,110 @@ TEST_F(TestOfflineLB, test_offlinelb_1) { } } +TEST_F(TestOfflineLB, test_offlinelb_2) { + using LBDataHolder = vt::vrt::collection::balance::LBDataHolder; + using ElementIDStruct = vt::vrt::collection::balance::ElementIDStruct; + using LoadSummary = vt::vrt::collection::balance::LoadSummary; + using LBDataRestartReader = vt::vrt::collection::balance::LBDataRestartReader; + + auto const this_node = theContext()->getNode(); + auto const num_nodes = theContext()->getNumNodes(); + auto const next_node = (this_node + 1) % num_nodes; + auto const prev_node = this_node - 1 >= 0 ? this_node - 1 : num_nodes - 1; + + std::unordered_map> ids; + int len = 2; + PhaseType num_phases = 10; + for (int i = 0; i < len * 2; i++) { + auto id = elm::ElmIDBits::createCollectionImpl(true, i+1, this_node, this_node); + id.curr_node = this_node; + ids[0].push_back(id); + id.curr_node = next_node; + ids[3].push_back(id); + id.curr_node = prev_node; + ids[6].push_back(id); + } + + for (int i = 0; i < len; i++) { + auto nid = elm::ElmIDBits::createCollectionImpl(true, i+1, next_node, this_node); + auto pid = elm::ElmIDBits::createCollectionImpl(true, i+1, prev_node, this_node); + ids[4].push_back(pid); + ids[7].push_back(nid); + } + + LBDataHolder dh; + for (PhaseType i = 0; i < num_phases; i++) { + if (i != 1 and i != 2 and i != 5 and i != 8 and i != 9) { + auto& elms = ids[i]; + for(std::size_t j = 0; j < elms.size(); j++) { + dh.node_data_[i][elms[j]] = LoadSummary{ static_cast(i + j) + 3}; + } + } + } + + using JSONAppender = util::json::Appender; + std::stringstream stream{std::ios_base::out | std::ios_base::in}; + nlohmann::json metadata, phasesMetadata; + phasesMetadata["count"] = num_phases; + phasesMetadata["skipped"]["list"] = {9}; + phasesMetadata["skipped"]["range"] = {{1,2}}; + phasesMetadata["identical_to_previous"]["list"] = {8}; + phasesMetadata["identical_to_previous"]["range"] = {{5,5}}; + metadata["type"] = "LBDatafile"; + metadata["phases"] = phasesMetadata; + + auto appender = std::make_unique( + "phases", metadata, std::move(stream), true + ); + for (PhaseType i = 0; i < num_phases; i++) { + // ignore skipped and identical phases + if(i != 1 and i != 2 and i != 5 and i != 8 and i != 9) { + auto j = dh.toJson(i); + appender->addElm(*j); + } + } + stream = appender->finish(); + + // Preapre configuration file + std::string file_name = getUniqueFilenameWithRanks(".txt"); + std::ofstream out(file_name); + + // Request OfflineLB for each phase. + // LBDataRestartReader will check beforehand if that phase requires OfflineLB. + out << "" + "0 OfflineLB\n" + "1 OfflineLB\n" + "2 OfflineLB\n" + "3 OfflineLB\n" + "4 OfflineLB\n" + "5 OfflineLB\n" + "6 OfflineLB\n" + "7 OfflineLB\n" + "8 OfflineLB\n" + "9 OfflineLB\n"; + out.close(); + + theConfig()->vt_lb = true; + theConfig()->vt_lb_file_name = file_name; + + auto up = LBDataRestartReader::construct(); + curRT->theLBDataReader = up.get(); + theLBDataReader()->readLBDataFromStream(std::move(stream)); + + vt::Index1D range{2*num_nodes}; + auto proxy = vt::makeCollection("simcol") + .bounds(range) + .bulkInsert() + .wait(); + + for (PhaseType i = 0; i < num_phases; i++) { + runInEpochCollective("run sparseHandler", [&]{ + proxy.broadcastCollective(i); + }); + thePhase()->nextPhaseCollective(); + } +} + #endif }}}} /* end namespace vt::tests::unit::lb */ diff --git a/tests/unit/runtime/test_initialization.cc b/tests/unit/runtime/test_initialization.cc index cdecb2fbfb..00309635f1 100644 --- a/tests/unit/runtime/test_initialization.cc +++ b/tests/unit/runtime/test_initialization.cc @@ -47,8 +47,12 @@ #include "test_helpers.h" #include +#include +#include +#include #include +#include namespace vt { namespace tests { namespace unit { @@ -320,4 +324,278 @@ TEST_F(TestInitialization, test_preconfigure_and_initialization) { vt::initializePreconfigured(&comm, &appConfig, vtConfig.get()); } +void prepareLBDataFiles(const std::string file_name_without_ext) { + using LBDataHolder = vt::vrt::collection::balance::LBDataHolder; + using ElementIDStruct = vt::vrt::collection::balance::ElementIDStruct; + using LoadSummary = vt::vrt::collection::balance::LoadSummary; + + auto const this_node = theContext()->getNode(); + auto const num_nodes = theContext()->getNumNodes(); + auto const next_node = (this_node + 1) % num_nodes; + auto const prev_node = this_node - 1 >= 0 ? this_node - 1 : num_nodes - 1; + + std::unordered_map> ids; + int len = 2; + PhaseType num_phases = 7; + for (int i = 0; i < len; i++) { + auto id = vt::elm::ElmIDBits::createCollectionImpl(true, i+1, this_node, this_node); + id.curr_node = this_node; + ids[0].push_back(id); + id.curr_node = next_node; + ids[3].push_back(id); + id.curr_node = prev_node; + ids[6].push_back(id); + } + + for (int i = 0; i < len; i++) { + auto pid = vt::elm::ElmIDBits::createCollectionImpl(true, i+1, prev_node, this_node); + auto nid = vt::elm::ElmIDBits::createCollectionImpl(true, i+1, next_node, this_node); + ids[1].push_back(pid); + ids[2].push_back(pid); + ids[4].push_back(nid); + ids[5].push_back(nid); + } + + LBDataHolder dh; + for (PhaseType i = 0; i < num_phases; i++) { + for (auto&& elm : ids[i]) { + dh.node_data_[i][elm] = LoadSummary{3}; + } + } + + using JSONAppender = util::json::Appender; + std::stringstream stream{std::ios_base::out | std::ios_base::in}; + nlohmann::json metadata; + metadata["type"] = "LBDatafile"; + auto w = std::make_unique( + "phases", metadata, std::move(stream), true + ); + for (PhaseType i = 0; i < num_phases; i++) { + auto j = dh.toJson(i); + w->addElm(*j); + } + stream = w->finish(); + + // save to file + std::string file_name = file_name_without_ext + "." + std::to_string(this_node) + ".json"; + std::filesystem::path file_path = std::filesystem::current_path() / file_name; + std::ofstream out(file_path); + out << stream.str(); + out.close(); +} + +TEST_F(TestInitialization, test_initialize_without_restart_reader) { + MPI_Comm comm = MPI_COMM_WORLD; + + static char prog_name[]{"vt_program"}; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "NoLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, false); + EXPECT_EQ(theConfig()->vt_lb_file_name, ""); + EXPECT_TRUE(theLBDataReader() == nullptr); +} + +#if vt_feature_cmake_lblite +TEST_F(TestInitialization, test_initialize_with_lb_data_in) { + MPI_Comm comm = MPI_COMM_WORLD; + + // Preapre data files + auto prefix = getUniqueFilenameWithRanks(); + prepareLBDataFiles(prefix); + + static char prog_name[]{"vt_program"}; + static char data_in[]{"--vt_lb_data_in"}; + std::string data_file_dir = "--vt_lb_data_dir_in="; + data_file_dir += std::filesystem::current_path(); + std::string data_file = "--vt_lb_data_file_in="; + data_file += prefix + ".%p.json"; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(data_in); + custom_args.emplace_back(const_cast(data_file_dir.c_str())); + custom_args.emplace_back(const_cast(data_file.c_str())); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "NoLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, true); + EXPECT_EQ(theConfig()->vt_lb_file_name, ""); + EXPECT_TRUE(theLBDataReader() == nullptr); +} + +TEST_F(TestInitialization, test_initialize_with_lb_data_and_config_offline_lb) { + MPI_Comm comm = MPI_COMM_WORLD; + + // Preapre data files + auto prefix = getUniqueFilenameWithRanks(); + prepareLBDataFiles(prefix); + + // Preapre configuration file + std::string file_name = getUniqueFilenameWithRanks(".txt"); + std::ofstream out(file_name); + out << "0 OfflineLB\n"; + out.close(); + + static char prog_name[]{"vt_program"}; + static char data_in[]{"--vt_lb_data_in"}; + std::string data_file_dir = "--vt_lb_data_dir_in="; + data_file_dir += std::filesystem::current_path(); + std::string data_file = "--vt_lb_data_file_in="; + data_file += prefix + ".%p.json"; + std::string config_file = "--vt_lb_file_name=" + file_name; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(data_in); + custom_args.emplace_back(const_cast(data_file_dir.c_str())); + custom_args.emplace_back(const_cast(data_file.c_str())); + custom_args.emplace_back(const_cast(config_file.c_str())); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "NoLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, true); + EXPECT_EQ(theConfig()->vt_lb_file_name, file_name); + EXPECT_TRUE(theLBDataReader() != nullptr); +} + +TEST_F(TestInitialization, test_initialize_with_lb_data_and_no_lb) { + MPI_Comm comm = MPI_COMM_WORLD; + + // Preapre data files + auto prefix = getUniqueFilenameWithRanks(); + prepareLBDataFiles(prefix); + + // Preapre configuration file + std::string file_name = getUniqueFilenameWithRanks(".txt"); + std::ofstream out(file_name); + out << "0 NoLB\n"; + out.close(); + + static char prog_name[]{"vt_program"}; + static char data_in[]{"--vt_lb_data_in"}; + std::string data_file_dir = "--vt_lb_data_dir_in="; + data_file_dir += std::filesystem::current_path(); + std::string data_file = "--vt_lb_data_file_in="; + data_file += prefix + ".%p.json"; + std::string config_file = "--vt_lb_file_name=" + file_name; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(data_in); + custom_args.emplace_back(const_cast(data_file_dir.c_str())); + custom_args.emplace_back(const_cast(data_file.c_str())); + custom_args.emplace_back(const_cast(config_file.c_str())); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "NoLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, true); + EXPECT_EQ(theConfig()->vt_lb_file_name, file_name); + EXPECT_TRUE(theLBDataReader() == nullptr); +} + +TEST_F(TestInitialization, test_initialize_with_lb_data_and_offline_lb) { + MPI_Comm comm = MPI_COMM_WORLD; + + // Preapre data files + auto prefix = getUniqueFilenameWithRanks(); + prepareLBDataFiles(prefix); + + static char prog_name[]{"vt_program"}; + static char data_in[]{"--vt_lb_data_in"}; + static char offline_lb[]{"--vt_lb_name=OfflineLB"}; + std::string data_file_dir = "--vt_lb_data_dir_in="; + data_file_dir += std::filesystem::current_path(); + std::string data_file = "--vt_lb_data_file_in="; + data_file += prefix + ".%p.json"; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(data_in); + custom_args.emplace_back(offline_lb); + custom_args.emplace_back(const_cast(data_file_dir.c_str())); + custom_args.emplace_back(const_cast(data_file.c_str())); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "OfflineLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, true); + EXPECT_EQ(theConfig()->vt_lb_file_name, ""); + EXPECT_TRUE(theLBDataReader() != nullptr); +} +#endif + +TEST_F(TestInitialization, test_initialize_with_lb_data_and_config_no_lb) { + MPI_Comm comm = MPI_COMM_WORLD; + + // Preapre data files + auto prefix = getUniqueFilenameWithRanks(); + prepareLBDataFiles(prefix); + + // Preapre configuration file + std::string file_name = getUniqueFilenameWithRanks(".txt"); + std::ofstream out(file_name); + out << "0 NoLB\n"; + out.close(); + + static char prog_name[]{"vt_program"}; + static char data_in[]{"--vt_lb_data_in"}; + std::string data_file_dir = "--vt_lb_data_dir_in="; + data_file_dir += std::filesystem::current_path(); + std::string data_file = "--vt_lb_data_file_in="; + data_file += prefix + ".%p.json"; + std::string config_file = "--vt_lb_file_name=" + file_name; + + std::vector custom_args; + custom_args.emplace_back(prog_name); + custom_args.emplace_back(data_in); + custom_args.emplace_back(const_cast(data_file_dir.c_str())); + custom_args.emplace_back(const_cast(data_file.c_str())); + custom_args.emplace_back(const_cast(config_file.c_str())); + custom_args.emplace_back(nullptr); + + int custom_argc = custom_args.size() - 1; + char** custom_argv = custom_args.data(); + + vt::initialize(custom_argc, custom_argv, &comm); + + EXPECT_EQ(theConfig()->prog_name, "vt_program"); + EXPECT_EQ(theConfig()->vt_lb_name, "NoLB"); + EXPECT_EQ(theConfig()->vt_lb_data_in, true); + EXPECT_EQ(theConfig()->vt_lb_file_name, file_name); + EXPECT_TRUE(theLBDataReader() == nullptr); +} + }}} // end namespace vt::tests::unit