From 031beadad7fdabe15e1e6fb598fea81006184c7e Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 23 Aug 2023 07:08:00 -0400 Subject: [PATCH 1/5] Reader-side Profiling --- source/adios2/engine/bp5/BP5Reader.cpp | 50 ++++++++++++++++++- source/adios2/engine/bp5/BP5Reader.h | 3 ++ .../toolkit/profiling/iochrono/IOChrono.cpp | 21 ++++++-- .../toolkit/profiling/iochrono/IOChrono.h | 1 + 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 687570fa87..679a0af97a 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -10,6 +10,7 @@ #include "BP5Reader.tcc" #include "adios2/helper/adiosMath.h" // SetWithinLimit +#include "adios2/toolkit/transport/file/FileFStream.h" #include #include @@ -31,7 +32,7 @@ namespace engine BP5Reader::BP5Reader(IO &io, const std::string &name, const Mode mode, helper::Comm comm) : Engine("BP5Reader", io, name, mode, std::move(comm)), m_MDFileManager(io, m_Comm), m_DataFileManager(io, m_Comm), m_MDIndexFileManager(io, m_Comm), - m_FileMetaMetadataManager(io, m_Comm), m_ActiveFlagFileManager(io, m_Comm), m_Remote() + m_FileMetaMetadataManager(io, m_Comm), m_ActiveFlagFileManager(io, m_Comm), m_Remote(), m_JSONProfiler(m_Comm) { PERFSTUBS_SCOPED_TIMER("BP5Reader::Open"); Init(); @@ -298,6 +299,7 @@ void BP5Reader::PerformLocalGets() // TP start = NOW(); PERFSTUBS_SCOPED_TIMER("BP5Reader::PerformGets"); + m_JSONProfiler.Start("DataRead"); size_t maxReadSize; // TP startGenerate = NOW(); @@ -317,6 +319,9 @@ void BP5Reader::PerformLocalGets() reqidx = nextRequest; ++nextRequest; } + if (reqidx <= nRequest) { + m_JSONProfiler.AddBytes("dataread", ReadRequests[reqidx].ReadLength); + } return reqidx; }; @@ -406,11 +411,13 @@ void BP5Reader::PerformLocalGets() { Req.DestinationAddr = buf.data(); } + m_JSONProfiler.AddBytes("dataread", Req.ReadLength); ReadData(m_DataFileManager, maxOpenFiles, Req.WriterRank, Req.Timestep, Req.StartOffset, Req.ReadLength, Req.DestinationAddr); m_BP5Deserializer->FinalizeGet(Req, false); } } + m_JSONProfiler.Stop("DataRead"); /*TP end = NOW(); double t1 = DURATION(start, end); double t2 = DURATION(startRead, end); @@ -789,16 +796,19 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol if (actualFileSize >= expectedMinFileSize) { + m_JSONProfiler.Start("MetaDataRead"); m_Metadata.Resize(fileFilteredSize, "allocating metadata buffer, " "in call to BP5Reader Open"); size_t mempos = 0; for (auto p : m_FilteredMetadataInfo) { + m_JSONProfiler.AddBytes("metadataread", p.second); m_MDFileManager.ReadFile(m_Metadata.m_Buffer.data() + mempos, p.second, p.first); mempos += p.second; } m_MDFileAlreadyReadSize = expectedMinFileSize; + m_JSONProfiler.Stop("MetaDataRead"); } else { @@ -824,12 +834,15 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol if (metametadataFileSize > m_MetaMetaDataFileAlreadyReadSize) { const size_t newMMDSize = metametadataFileSize - m_MetaMetaDataFileAlreadyReadSize; + m_JSONProfiler.Start("MetaMetaDataRead"); + m_JSONProfiler.AddBytes("metametadataread", newMMDSize); m_MetaMetadata.Resize(metametadataFileSize, "(re)allocating meta-meta-data buffer, " "in call to BP5Reader Open"); m_FileMetaMetadataManager.ReadFile(m_MetaMetadata.m_Buffer.data() + m_MetaMetaDataFileAlreadyReadSize, newMMDSize, m_MetaMetaDataFileAlreadyReadSize); m_MetaMetaDataFileAlreadyReadSize += newMMDSize; + m_JSONProfiler.Stop("MetaMetaDataRead"); } } @@ -1194,6 +1207,7 @@ void BP5Reader::DoClose(const int transportIndex) { EndStep(); } + FlushProfiler(); m_DataFileManager.CloseFiles(); m_MDFileManager.CloseFiles(); m_MDIndexFileManager.CloseFiles(); @@ -1204,6 +1218,40 @@ void BP5Reader::DoClose(const int transportIndex) } } +#if defined(_WIN32) +# include +# define getpid() GetCurrentProcessId(); +#elif defined(__linux) || defined(__APPLE__) || defined(__OpenBSD__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ + defined(__CYGWIN__) +#else +# define getpid() (long long) -1; +#endif + +void BP5Reader::FlushProfiler() +{ + + auto LineJSON = m_JSONProfiler.GetRankProfilingJSON({}, {}); + const std::vector profilingJSON(m_JSONProfiler.AggregateProfilingJSON(LineJSON)); + + if (m_RankMPI == 0) + { + std::string profileFileName; + transport::FileFStream profilingJSONStream(m_Comm); + std::string bpBaseName = m_Name; + + auto PID = getpid(); + std::stringstream PIDstr; + PIDstr << std::hex << PID; + // write profile json in /tmp + profileFileName = "/tmp/" + bpBaseName + "_" + PIDstr.str() + "_profiling.json"; + + profilingJSONStream.Open(profileFileName, Mode::Write); + profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); + profilingJSONStream.Close(); + } +} + size_t BP5Reader::DoSteps() const { return m_StepsCount; } void BP5Reader::NotifyEngineNoVarsQuery() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 7d837184ac..117a05f3c2 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -58,6 +58,7 @@ class BP5Reader : public BP5Engine, public Engine MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; bool VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const; bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax); + adios2::profiling::JSONProfiler m_JSONProfiler; private: format::BP5Deserializer *m_BP5Deserializer = nullptr; @@ -200,6 +201,8 @@ class BP5Reader : public BP5Engine, public Engine void DoClose(const int transportIndex = -1) final; + void FlushProfiler(); + void GetSyncCommon(VariableBase &variable, void *data); void GetDeferredCommon(VariableBase &variable, void *data); diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp index 40fb740aa2..5c3761f7dc 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp @@ -57,6 +57,12 @@ JSONProfiler::JSONProfiler(helper::Comm const &comm) : m_Comm(comm) AddTimerWatch("PDW"); m_Profiler.m_Bytes.emplace("buffering", 0); + AddTimerWatch("DataRead"); + m_Profiler.m_Bytes.emplace("dataread", 0); + AddTimerWatch("MetaDataRead"); + m_Profiler.m_Bytes.emplace("metadataread", 0); + AddTimerWatch("MetaMetaDataRead"); + m_Profiler.m_Bytes.emplace("metadmetaataread", 0); m_RankMPI = m_Comm.Rank(); } @@ -92,11 +98,20 @@ std::string JSONProfiler::GetRankProfilingJSON( for (const auto &timerPair : profiler.m_Timers) { const profiling::Timer &timer = timerPair.second; - // rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + - // "\": " + std::to_string(timer.m_ProcessTime) + ", "; - timer.AddToJsonStr(rankLog); + if (timer.m_nCalls > 0) { + rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + + "\": " + std::to_string(timer.m_ProcessTime) + ", "; + timer.AddToJsonStr(rankLog); + } } + size_t DataBytes = m_Profiler.m_Bytes["dataread"]; + size_t MetaDataBytes = m_Profiler.m_Bytes["metadataread"]; + size_t MetaMetaDataBytes = m_Profiler.m_Bytes["metametadataread"]; + rankLog += ", \"databytes\":" + std::to_string(DataBytes); + rankLog += ", \"metadatabytes\":" + std::to_string(MetaDataBytes); + rankLog += ", \"metametadatabytes\":" + std::to_string(MetaMetaDataBytes); + const size_t transportsSize = transportsTypes.size(); for (unsigned int t = 0; t < transportsSize; ++t) diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.h b/source/adios2/toolkit/profiling/iochrono/IOChrono.h index 735d4fea1c..a2d410fccd 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.h +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.h @@ -68,6 +68,7 @@ class JSONProfiler void Start(const std::string process) { m_Profiler.Start(process); }; void Stop(const std::string process) { m_Profiler.Stop(process); }; + void AddBytes(const std::string process, size_t bytes) { m_Profiler.m_Bytes[process] += bytes ;}; std::string GetRankProfilingJSON( const std::vector &transportsTypes, From cb28597135f12bd8d37082aec7109b5b947a1c50 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 23 Aug 2023 07:12:41 -0400 Subject: [PATCH 2/5] format --- source/adios2/engine/bp5/BP5Reader.cpp | 61 ++++++++++--------- .../toolkit/profiling/iochrono/IOChrono.cpp | 11 ++-- .../toolkit/profiling/iochrono/IOChrono.h | 5 +- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 679a0af97a..2ce49fd991 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -32,7 +32,8 @@ namespace engine BP5Reader::BP5Reader(IO &io, const std::string &name, const Mode mode, helper::Comm comm) : Engine("BP5Reader", io, name, mode, std::move(comm)), m_MDFileManager(io, m_Comm), m_DataFileManager(io, m_Comm), m_MDIndexFileManager(io, m_Comm), - m_FileMetaMetadataManager(io, m_Comm), m_ActiveFlagFileManager(io, m_Comm), m_Remote(), m_JSONProfiler(m_Comm) + m_FileMetaMetadataManager(io, m_Comm), m_ActiveFlagFileManager(io, m_Comm), m_Remote(), + m_JSONProfiler(m_Comm) { PERFSTUBS_SCOPED_TIMER("BP5Reader::Open"); Init(); @@ -319,9 +320,10 @@ void BP5Reader::PerformLocalGets() reqidx = nextRequest; ++nextRequest; } - if (reqidx <= nRequest) { - m_JSONProfiler.AddBytes("dataread", ReadRequests[reqidx].ReadLength); - } + if (reqidx <= nRequest) + { + m_JSONProfiler.AddBytes("dataread", ReadRequests[reqidx].ReadLength); + } return reqidx; }; @@ -411,7 +413,7 @@ void BP5Reader::PerformLocalGets() { Req.DestinationAddr = buf.data(); } - m_JSONProfiler.AddBytes("dataread", Req.ReadLength); + m_JSONProfiler.AddBytes("dataread", Req.ReadLength); ReadData(m_DataFileManager, maxOpenFiles, Req.WriterRank, Req.Timestep, Req.StartOffset, Req.ReadLength, Req.DestinationAddr); m_BP5Deserializer->FinalizeGet(Req, false); @@ -796,19 +798,19 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol if (actualFileSize >= expectedMinFileSize) { - m_JSONProfiler.Start("MetaDataRead"); + m_JSONProfiler.Start("MetaDataRead"); m_Metadata.Resize(fileFilteredSize, "allocating metadata buffer, " "in call to BP5Reader Open"); size_t mempos = 0; for (auto p : m_FilteredMetadataInfo) { - m_JSONProfiler.AddBytes("metadataread", p.second); + m_JSONProfiler.AddBytes("metadataread", p.second); m_MDFileManager.ReadFile(m_Metadata.m_Buffer.data() + mempos, p.second, p.first); mempos += p.second; } m_MDFileAlreadyReadSize = expectedMinFileSize; - m_JSONProfiler.Stop("MetaDataRead"); + m_JSONProfiler.Stop("MetaDataRead"); } else { @@ -834,15 +836,15 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol if (metametadataFileSize > m_MetaMetaDataFileAlreadyReadSize) { const size_t newMMDSize = metametadataFileSize - m_MetaMetaDataFileAlreadyReadSize; - m_JSONProfiler.Start("MetaMetaDataRead"); - m_JSONProfiler.AddBytes("metametadataread", newMMDSize); + m_JSONProfiler.Start("MetaMetaDataRead"); + m_JSONProfiler.AddBytes("metametadataread", newMMDSize); m_MetaMetadata.Resize(metametadataFileSize, "(re)allocating meta-meta-data buffer, " "in call to BP5Reader Open"); m_FileMetaMetadataManager.ReadFile(m_MetaMetadata.m_Buffer.data() + m_MetaMetaDataFileAlreadyReadSize, newMMDSize, m_MetaMetaDataFileAlreadyReadSize); m_MetaMetaDataFileAlreadyReadSize += newMMDSize; - m_JSONProfiler.Stop("MetaMetaDataRead"); + m_JSONProfiler.Stop("MetaMetaDataRead"); } } @@ -1219,36 +1221,35 @@ void BP5Reader::DoClose(const int transportIndex) } #if defined(_WIN32) -# include -# define getpid() GetCurrentProcessId(); -#elif defined(__linux) || defined(__APPLE__) || defined(__OpenBSD__) || \ - defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ - defined(__CYGWIN__) +#include +#define getpid() GetCurrentProcessId(); +#elif defined(__linux) || defined(__APPLE__) || defined(__OpenBSD__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) #else -# define getpid() (long long) -1; +#define getpid() (long long)-1; #endif void BP5Reader::FlushProfiler() { - auto LineJSON = m_JSONProfiler.GetRankProfilingJSON({}, {}); + auto LineJSON = m_JSONProfiler.GetRankProfilingJSON({}, {}); const std::vector profilingJSON(m_JSONProfiler.AggregateProfilingJSON(LineJSON)); if (m_RankMPI == 0) { std::string profileFileName; - transport::FileFStream profilingJSONStream(m_Comm); - std::string bpBaseName = m_Name; - - auto PID = getpid(); - std::stringstream PIDstr; - PIDstr << std::hex << PID; - // write profile json in /tmp - profileFileName = "/tmp/" + bpBaseName + "_" + PIDstr.str() + "_profiling.json"; - - profilingJSONStream.Open(profileFileName, Mode::Write); - profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); - profilingJSONStream.Close(); + transport::FileFStream profilingJSONStream(m_Comm); + std::string bpBaseName = m_Name; + + auto PID = getpid(); + std::stringstream PIDstr; + PIDstr << std::hex << PID; + // write profile json in /tmp + profileFileName = "/tmp/" + bpBaseName + "_" + PIDstr.str() + "_profiling.json"; + + profilingJSONStream.Open(profileFileName, Mode::Write); + profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); + profilingJSONStream.Close(); } } diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp index 5c3761f7dc..a8713339ed 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp @@ -98,11 +98,12 @@ std::string JSONProfiler::GetRankProfilingJSON( for (const auto &timerPair : profiler.m_Timers) { const profiling::Timer &timer = timerPair.second; - if (timer.m_nCalls > 0) { - rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + - "\": " + std::to_string(timer.m_ProcessTime) + ", "; - timer.AddToJsonStr(rankLog); - } + if (timer.m_nCalls > 0) + { + rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + + "\": " + std::to_string(timer.m_ProcessTime) + ", "; + timer.AddToJsonStr(rankLog); + } } size_t DataBytes = m_Profiler.m_Bytes["dataread"]; diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.h b/source/adios2/toolkit/profiling/iochrono/IOChrono.h index a2d410fccd..9abe786144 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.h +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.h @@ -68,7 +68,10 @@ class JSONProfiler void Start(const std::string process) { m_Profiler.Start(process); }; void Stop(const std::string process) { m_Profiler.Stop(process); }; - void AddBytes(const std::string process, size_t bytes) { m_Profiler.m_Bytes[process] += bytes ;}; + void AddBytes(const std::string process, size_t bytes) + { + m_Profiler.m_Bytes[process] += bytes; + }; std::string GetRankProfilingJSON( const std::vector &transportsTypes, From 7f3cecfa40ab25a5b57542067e520c525f4c9134 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 23 Aug 2023 12:51:16 -0400 Subject: [PATCH 3/5] Remove file before writing, catch exceptions --- source/adios2/engine/bp5/BP5Reader.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 2ce49fd991..8062f9184a 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -14,7 +14,9 @@ #include #include +#include #include +#include #include #include @@ -1247,9 +1249,16 @@ void BP5Reader::FlushProfiler() // write profile json in /tmp profileFileName = "/tmp/" + bpBaseName + "_" + PIDstr.str() + "_profiling.json"; - profilingJSONStream.Open(profileFileName, Mode::Write); - profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); - profilingJSONStream.Close(); + try + { + (void)remove(profileFileName.c_str()); + profilingJSONStream.Open(profileFileName, Mode::Write); + profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); + profilingJSONStream.Close(); + } + catch (...) + { // do nothing + } } } From 5ae2babca2d84f5a848fa38eed77a86f48244f24 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Wed, 23 Aug 2023 18:24:37 -0400 Subject: [PATCH 4/5] reorder --- source/adios2/engine/bp5/BP5Reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 117a05f3c2..3c45b1db2c 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -58,7 +58,6 @@ class BP5Reader : public BP5Engine, public Engine MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; bool VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const; bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax); - adios2::profiling::JSONProfiler m_JSONProfiler; private: format::BP5Deserializer *m_BP5Deserializer = nullptr; @@ -96,6 +95,7 @@ class BP5Reader : public BP5Engine, public Engine transportman::TransportMan m_ActiveFlagFileManager; Remote m_Remote; bool m_WriterIsActive = true; + adios2::profiling::JSONProfiler m_JSONProfiler; /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; From 66f24a3f396732c669db569a73dcdd5dbc98f998 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Thu, 24 Aug 2023 11:11:07 -0400 Subject: [PATCH 5/5] defines --- source/adios2/engine/bp5/BP5Reader.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 8062f9184a..fd2f4c33db 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -1227,6 +1227,8 @@ void BP5Reader::DoClose(const int transportIndex) #define getpid() GetCurrentProcessId(); #elif defined(__linux) || defined(__APPLE__) || defined(__OpenBSD__) || defined(__FreeBSD__) || \ defined(__NetBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) +#include +#include #else #define getpid() (long long)-1; #endif