diff --git a/src/coreComponents/managers/EventManager.cpp b/src/coreComponents/managers/EventManager.cpp index 1cdbbf56e04..32d82075077 100644 --- a/src/coreComponents/managers/EventManager.cpp +++ b/src/coreComponents/managers/EventManager.cpp @@ -166,6 +166,16 @@ void EventManager::Run(dataRepository::ManagedGroup * domain) integer const verbosity = this->getReference(viewKeys.verbosity); integer exitFlag = 0; + // Setup MPI communication + integer rank = 0; + integer comm_size = 1; + #if USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &comm_size); + #endif + real64 send_buffer[2]; + array1d receive_buffer(2 * comm_size); + // Setup event targets this->forSubGroups([]( EventBase * subEvent ) -> void { @@ -176,13 +186,20 @@ void EventManager::Run(dataRepository::ManagedGroup * domain) while((time < maxTime) && (cycle < maxCycle) && (exitFlag == 0)) { real64 nextDt = std::numeric_limits::max(); - std::cout << "Time: " << time << "s, dt:" << dt << "s, Cycle: " << cycle << std::endl; - + if (rank == 0) + { + std::cout << "Time: " << time << "s, dt:" << dt << "s, Cycle: " << cycle << std::endl; + } + this->forSubGroups([&]( EventBase * subEvent ) -> void { + // Calculate the event and sub-event forecasts + // Note: because events can be nested, the mpi reduce for event + // forecasts need to happen in EventBase. subEvent->CheckEvents(time, dt, cycle, domain); integer eventForecast = subEvent->GetForecast(); + // Execute, signal events if (eventForecast == 1) { subEvent->SignalToPrepareForExecution(time, dt, cycle, domain); @@ -193,19 +210,20 @@ void EventManager::Run(dataRepository::ManagedGroup * domain) subEvent->Execute(time, dt, cycle, domain); } - real64 requestedDt = 1e6; + // Estimate the time-step for the next cycle if (eventForecast <= 1) { - requestedDt = subEvent->GetTimestepRequest(time + dt); + real64 requestedDt = subEvent->GetTimestepRequest(time + dt); + nextDt = std::min(requestedDt, nextDt); } - nextDt = std::min(requestedDt, nextDt); + // Check the exit flag exitFlag += subEvent->GetExitFlag(); - + // Debug information - if (verbosity > 0) + if ((verbosity > 0) && (rank == 0)) { - std::cout << " Event: " << subEvent->getName() << ", f=" << eventForecast << ", dt_r=" << requestedDt << std::endl; + std::cout << " Event: " << subEvent->getName() << ", f=" << eventForecast << std::endl; } }); @@ -213,11 +231,37 @@ void EventManager::Run(dataRepository::ManagedGroup * domain) ++cycle; dt = nextDt; dt = (time + dt > maxTime) ? (maxTime - time) : dt; + + #if USE_MPI + send_buffer[0] = dt; + send_buffer[1] = static_cast(exitFlag); + MPI_Gather(send_buffer, 2, MPI_DOUBLE, receive_buffer.data(), 2, MPI_DOUBLE, 0, MPI_COMM_WORLD); + + if (rank == 0) + { + for (integer ii=0; ii 0.5) + { + exitFlag = 1; + } + #endif } // Cleanup - std::cout << "Cleaning up events" << std::endl; + if (rank == 0) + { + std::cout << "Cleaning up events" << std::endl; + } + this->forSubGroups([&]( EventBase * subEvent ) -> void { subEvent->Cleanup(time, cycle, domain); diff --git a/src/coreComponents/managers/Events/EventBase.cpp b/src/coreComponents/managers/Events/EventBase.cpp index c60023944b1..0ceefc550b9 100644 --- a/src/coreComponents/managers/Events/EventBase.cpp +++ b/src/coreComponents/managers/Events/EventBase.cpp @@ -88,7 +88,7 @@ void EventBase::FillDocumentationNode() "real64", "end time", "end time", - "1.0e9", + "1.0e100", "", 0, 1, @@ -306,6 +306,8 @@ void EventBase::Step(real64 const time, integer const cycle, dataRepository::ManagedGroup * domain ) { + // Note: do we need an mpi barrier here? + if (m_target != nullptr) { m_target->Execute(time, dt, cycle, domain); @@ -377,4 +379,15 @@ void EventBase::Cleanup(real64 const& time_n, +integer EventBase::GetExitFlag() +{ + this->forSubGroups([&]( EventBase * subEvent ) -> void + { + m_exitFlag += subEvent->GetExitFlag(); + }); + + return m_exitFlag; +} + + } /* namespace geosx */ diff --git a/src/coreComponents/managers/Events/EventBase.hpp b/src/coreComponents/managers/Events/EventBase.hpp index ccfd46fb25c..58c839de5cd 100644 --- a/src/coreComponents/managers/Events/EventBase.hpp +++ b/src/coreComponents/managers/Events/EventBase.hpp @@ -160,13 +160,12 @@ class EventBase : public ExecutableGroup integer GetForecast(){ return m_eventForecast; } void SetForecast(integer forecast){ m_eventForecast = forecast; } - integer GetExitFlag(){ return m_exitFlag; } + integer GetExitFlag(); void SetExitFlag(integer flag){ m_exitFlag = flag; } private: integer m_eventForecast = 0; integer m_exitFlag = 0; - }; } /* namespace geosx */ diff --git a/src/coreComponents/managers/Events/HaltEvent.cpp b/src/coreComponents/managers/Events/HaltEvent.cpp index f8bebad6c41..bbc10514f66 100644 --- a/src/coreComponents/managers/Events/HaltEvent.cpp +++ b/src/coreComponents/managers/Events/HaltEvent.cpp @@ -86,6 +86,14 @@ void HaltEvent::EstimateEventTiming(real64 const time, m_realDt = currentTime - m_lastTime; m_lastTime = currentTime; integer forecast = static_cast((maxRuntime - (currentTime - m_startTime)) / m_realDt); + + // The timing for the ranks may differ slightly, so synchronize + #if USE_MPI + integer forecast_global; + MPI_Allreduce(&forecast, &forecast_global, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + forecast = forecast_global; + #endif + SetForecast(forecast); if (this->GetForecast() <= 0) diff --git a/src/coreComponents/managers/Events/PeriodicEvent.cpp b/src/coreComponents/managers/Events/PeriodicEvent.cpp index bb3980b8e4e..6dea502e16c 100644 --- a/src/coreComponents/managers/Events/PeriodicEvent.cpp +++ b/src/coreComponents/managers/Events/PeriodicEvent.cpp @@ -245,6 +245,14 @@ void PeriodicEvent::CheckOptionalFunctionThreshold(real64 const time, // Find the function (min, average, max) real64_array stats = function->EvaluateStats(m_functionTarget, time, mySet); result = stats[functionStatOption]; + + // Because the function applied to an object may differ by rank, synchronize + // (Note: this shouldn't occur very often, since it is only called if the base forecast <= 0) + #if USE_MPI + real64 result_global; + MPI_Allreduce(&result, &result_global, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + result = result_global; + #endif } // Forcast event diff --git a/src/coreComponents/physicsSolvers/CMakeLists.txt b/src/coreComponents/physicsSolvers/CMakeLists.txt index 87cf67a0e71..ae9a088d578 100644 --- a/src/coreComponents/physicsSolvers/CMakeLists.txt +++ b/src/coreComponents/physicsSolvers/CMakeLists.txt @@ -8,6 +8,7 @@ set(physicsSolvers_headers FiniteVolume/SinglePhaseFlow.hpp SimpleSolvers/LaplaceFEM.hpp src/SolidMechanicsLagrangianFEM.hpp + src/DummySolver.hpp ) # @@ -19,6 +20,7 @@ set(physicsSolvers_sources FiniteVolume/SinglePhaseFlow.cpp SimpleSolvers/LaplaceFEM.cpp src/SolidMechanicsLagrangianFEM.cpp + src/DummySolver.cpp ) diff --git a/src/coreComponents/physicsSolvers/src/DummySolver.cpp b/src/coreComponents/physicsSolvers/src/DummySolver.cpp new file mode 100644 index 00000000000..9eb20e4587b --- /dev/null +++ b/src/coreComponents/physicsSolvers/src/DummySolver.cpp @@ -0,0 +1,107 @@ +/* + *~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * LLNL-CODE-746361 + * + * All rights reserved. See COPYRIGHT for details. + * + * This file is part of the GEOSX Simulation Framework. + * + * GEOSX is a free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License (as published by the + * Free Software Foundation) version 2.1 dated February 1999. + *~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + + +#include "DummySolver.hpp" +#include "dataRepository/ManagedGroup.hpp" +#include +#include + +namespace geosx +{ + + +using namespace dataRepository; + + +DummySolver::DummySolver( const std::string& name, + ManagedGroup * const parent ): + SolverBase( name, parent ) +{} + + + +DummySolver::~DummySolver() +{ + // TODO Auto-generated destructor stub +} + + +void DummySolver::FillDocumentationNode() +{ + cxx_utilities::DocumentationNode * const docNode = this->getDocumentationNode(); + SolverBase::FillDocumentationNode(); + + docNode->setName(this->CatalogName()); + docNode->setSchemaType("Node"); + docNode->setShortDescription("Dummy solver for testing time-stepping behavior"); + + docNode->AllocateChildNode( viewKeys.rand_scale.Key(), + viewKeys.rand_scale.Key(), + -1, + "real64", + "real64", + "Scale for modifying requested dt", + "Scale for modifying requested dt", + "1e-9", + "", + 1, + 1, + 0 ); + +} + + +void DummySolver::Initialize( ManagedGroup * const problemManager ) +{ + integer rank = 0; + #if USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + #endif + std::srand(rank * 12345); +} + + +real64 DummySolver::SolverStep( real64 const& time_n, + real64 const& dt, + const int cycleNumber, + DomainPartition * domain ) +{ + std::this_thread::sleep_for(std::chrono::seconds(1)); + return dt; +} + + +real64 DummySolver::GetTimestepRequest(real64 const time) +{ + integer rank = 0; + #if USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + #endif + + real64 const rand_scale = this->getReference(viewKeys.rand_scale); + real64 dt_request = std::rand() * rand_scale; + + std::cout << "time=" << time << ", solver=" << this->getName() << ", rank=" << rank << ", dt_r=" << dt_request << std::endl; + + return dt_request; +} + + +REGISTER_CATALOG_ENTRY( SolverBase, DummySolver, std::string const &, ManagedGroup * const ) +} /* namespace ANST */ diff --git a/src/coreComponents/physicsSolvers/src/DummySolver.hpp b/src/coreComponents/physicsSolvers/src/DummySolver.hpp new file mode 100644 index 00000000000..2ead93256d5 --- /dev/null +++ b/src/coreComponents/physicsSolvers/src/DummySolver.hpp @@ -0,0 +1,64 @@ +/* + *~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * + * Produced at the Lawrence Livermore National Laboratory + * + * LLNL-CODE-746361 + * + * All rights reserved. See COPYRIGHT for details. + * + * This file is part of the GEOSX Simulation Framework. + * + * GEOSX is a free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License (as published by the + * Free Software Foundation) version 2.1 dated February 1999. + *~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + + +#ifndef DUMMYSOLVER_HPP_ +#define DUMMYSOLVER_HPP_ + +#include "physicsSolvers/SolverBase.hpp" + +namespace geosx +{ +namespace dataRepository +{ +class ManagedGroup; +} +class DomainPartition; + +class DummySolver : public SolverBase +{ +public: + DummySolver( const std::string& name, + ManagedGroup * const parent ); + + + virtual ~DummySolver() override; + + static string CatalogName() { return "DummySolver"; } + + virtual void FillDocumentationNode() override; + + virtual void Initialize( ManagedGroup * const problemManager ) override final; + + virtual real64 SolverStep( real64 const& time_n, + real64 const& dt, + integer const cycleNumber, + DomainPartition * domain ) override; + + virtual real64 GetTimestepRequest(real64 const time) override; + + struct viewKeysStruct + { + dataRepository::ViewKey rand_scale = { "rand_scale" }; + } viewKeys; + +}; + +} /* namespace geosx */ + +#endif /* DUMMYSOLVER_HPP_ */