serializer and file engine

source/adios2/CMakeLists.txt

@@ -56,6 +56,10 @@ add_library(adios2_core
   engine/bp4/BP4Reader.cpp engine/bp4/BP4Reader.tcc
   engine/bp4/BP4Writer.cpp engine/bp4/BP4Writer.tcc
 
+  engine/bp5/BP5Engine.cpp
+  engine/bp5/BP5Reader.cpp engine/bp5/BP5Reader.tcc
+  engine/bp5/BP5Writer.cpp engine/bp5/BP5Writer.tcc
+
   engine/skeleton/SkeletonReader.cpp engine/skeleton/SkeletonReader.tcc
   engine/skeleton/SkeletonWriter.cpp engine/skeleton/SkeletonWriter.tcc
 
@@ -67,7 +71,9 @@ add_library(adios2_core
   engine/nullcore/NullCoreWriter.cpp engine/nullcore/NullCoreWriter.tcc
 #toolkit
   toolkit/format/buffer/Buffer.cpp
+  toolkit/format/buffer/BufferV.cpp
   toolkit/format/buffer/heap/BufferSTL.cpp
+  toolkit/format/buffer/ffs/BufferFFS.cpp
 
   toolkit/format/bp/BPBase.cpp toolkit/format/bp/BPBase.tcc
   toolkit/format/bp/BPSerializer.cpp toolkit/format/bp/BPSerializer.tcc
@@ -120,7 +126,7 @@ target_include_directories(adios2_core
     $<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source>
     $<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source>
 )
-target_link_libraries(adios2_core PRIVATE adios2sys_interface adios2::thirdparty::pugixml taustubs adios2::thirdparty::yaml-cpp)
+target_link_libraries(adios2_core PRIVATE adios2sys_interface adios2::thirdparty::pugixml taustubs adios2::thirdparty::yaml-cpp ffs::ffs)
 target_link_libraries(adios2_core PUBLIC ${CMAKE_THREAD_LIBS_INIT})
 target_compile_features(adios2_core PUBLIC "$<BUILD_INTERFACE:${ADIOS2_CXX11_FEATURES}>")
 
@@ -210,6 +216,11 @@ endif()
 if(ADIOS2_HAVE_SST)
   add_subdirectory(toolkit/sst)
   target_sources(adios2_core PRIVATE
+
+    toolkit/format/bp5/BP5Base.cpp
+    toolkit/format/bp5/BP5Serializer.cpp
+    toolkit/format/bp5/BP5Deserializer.cpp  toolkit/format/bp5/BP5Deserializer.tcc
+
     engine/sst/SstReader.cpp
     engine/sst/SstWriter.cpp
     engine/sst/SstParamParser.cpp

source/adios2/core/IO.cpp

@@ -22,6 +22,8 @@
 #include "adios2/engine/bp3/BP3Writer.h"
 #include "adios2/engine/bp4/BP4Reader.h"
 #include "adios2/engine/bp4/BP4Writer.h"
+#include "adios2/engine/bp5/BP5Reader.h"
+#include "adios2/engine/bp5/BP5Writer.h"
 #include "adios2/engine/inline/InlineReader.h"
 #include "adios2/engine/inline/InlineWriter.h"
 #include "adios2/engine/null/NullEngine.h"
@@ -65,6 +67,8 @@ std::unordered_map<std::string, IO::EngineFactoryEntry> Factory = {
      {IO::MakeEngine<engine::BP3Reader>, IO::MakeEngine<engine::BP3Writer>}},
     {"bp4",
      {IO::MakeEngine<engine::BP4Reader>, IO::MakeEngine<engine::BP4Writer>}},
+    {"bp5",
+     {IO::MakeEngine<engine::BP5Reader>, IO::MakeEngine<engine::BP5Writer>}},
     {"hdfmixer",
 #ifdef ADIOS2_HAVE_HDF5
      IO_MakeEngine_HDFMixer()

source/adios2/engine/bp5/BP5Engine.cpp

@@ -0,0 +1,188 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * BP5Engine.cpp
+ *
+ */
+
+#include "BP5Engine.h"
+
+#include "adios2/common/ADIOSMacros.h"
+#include "adios2/common/ADIOSTypes.h" //PathSeparator
+#include "adios2/core/IO.h"
+#include "adios2/helper/adiosFunctions.h" //CreateDirectory, StringToTimeUnit,
+
+#include <ctime>
+#include <iostream>
+
+namespace adios2
+{
+namespace core
+{
+namespace engine
+{
+
+std::vector<std::string>
+BP5Engine::GetBPMetadataFileNames(const std::vector<std::string> &names) const
+    noexcept
+{
+    std::vector<std::string> metadataFileNames;
+    metadataFileNames.reserve(names.size());
+    for (const auto &name : names)
+    {
+        metadataFileNames.push_back(GetBPMetadataFileName(name));
+    }
+    return metadataFileNames;
+}
+
+std::vector<std::string> BP5Engine::GetBPMetaMetadataFileNames(
+    const std::vector<std::string> &names) const noexcept
+{
+    std::vector<std::string> metaMetadataFileNames;
+    metaMetadataFileNames.reserve(names.size());
+    for (const auto &name : names)
+    {
+        metaMetadataFileNames.push_back(GetBPMetaMetadataFileName(name));
+    }
+    return metaMetadataFileNames;
+}
+
+std::string BP5Engine::GetBPMetadataFileName(const std::string &name) const
+    noexcept
+{
+    const std::string bpName = helper::RemoveTrailingSlash(name);
+    const size_t index = 0; // global metadata file is generated by rank 0
+    /* the name of the metadata file is "md.0" */
+    const std::string bpMetaDataRankName(bpName + PathSeparator + "md." +
+                                         std::to_string(index));
+    return bpMetaDataRankName;
+}
+
+std::string BP5Engine::GetBPMetaMetadataFileName(const std::string &name) const
+    noexcept
+{
+    const std::string bpName = helper::RemoveTrailingSlash(name);
+    const size_t index = 0; // global metadata file is generated by rank 0
+    /* the name of the metadata file is "md.0" */
+    const std::string bpMetaMetaDataRankName(bpName + PathSeparator + "mmd." +
+                                             std::to_string(index));
+    return bpMetaMetaDataRankName;
+}
+
+std::vector<std::string> BP5Engine::GetBPMetadataIndexFileNames(
+    const std::vector<std::string> &names) const noexcept
+{
+    std::vector<std::string> metadataIndexFileNames;
+    metadataIndexFileNames.reserve(names.size());
+    for (const auto &name : names)
+    {
+        metadataIndexFileNames.push_back(GetBPMetadataIndexFileName(name));
+    }
+    return metadataIndexFileNames;
+}
+
+std::string BP5Engine::GetBPMetadataIndexFileName(const std::string &name) const
+    noexcept
+{
+    const std::string bpName = helper::RemoveTrailingSlash(name);
+    /* the name of the metadata index file is "md.idx" */
+    const std::string bpMetaDataIndexRankName(bpName + PathSeparator +
+                                              "md.idx");
+    return bpMetaDataIndexRankName;
+}
+
+std::string BP5Engine::GetBPSubStreamName(const std::string &name,
+                                          const size_t id,
+                                          const bool hasSubFiles,
+                                          const bool isReader) const noexcept
+{
+    if (!hasSubFiles)
+    {
+        return name;
+    }
+
+    const std::string bpName = helper::RemoveTrailingSlash(name);
+
+    const size_t index = id;
+    //    isReader ? id
+    //	: m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : id;
+
+    /* the name of a data file starts with "data." */
+    const std::string bpRankName(bpName + PathSeparator + "data." +
+                                 std::to_string(index));
+    return bpRankName;
+}
+
+std::vector<std::string>
+BP5Engine::GetBPSubStreamNames(const std::vector<std::string> &names) const
+    noexcept
+{
+    std::vector<std::string> bpNames;
+    bpNames.reserve(names.size());
+
+    for (const auto &name : names)
+    {
+        bpNames.push_back(
+            GetBPSubStreamName(name, static_cast<unsigned int>(m_RankMPI)));
+    }
+    return bpNames;
+}
+
+void BP5Engine::ParseParams(IO &io, struct BP5Params &Params)
+{
+    std::memset(&Params, 0, sizeof(Params));
+
+    auto lf_SetBoolParameter = [&](const std::string key, bool &parameter) {
+        auto itKey = io.m_Parameters.find(key);
+        if (itKey != io.m_Parameters.end())
+        {
+            std::string value = itKey->second;
+            std::transform(value.begin(), value.end(), value.begin(),
+                           ::tolower);
+            if (value == "yes" || value == "true" || value == "on")
+            {
+                parameter = true;
+            }
+            else if (value == "no" || value == "false" || value == "off")
+            {
+                parameter = false;
+            }
+            else
+            {
+                throw std::invalid_argument(
+                    "ERROR: Unknown BP5 Boolean parameter \"" + value + "\"");
+            }
+        }
+    };
+    auto lf_SetIntParameter = [&](const std::string key, int &parameter) {
+        auto itKey = io.m_Parameters.find(key);
+        if (itKey != io.m_Parameters.end())
+        {
+            parameter = std::stoi(itKey->second);
+            return true;
+        }
+        return false;
+    };
+
+    auto lf_SetStringParameter = [&](const std::string key,
+                                     std::string &parameter) {
+        auto itKey = io.m_Parameters.find(key);
+        if (itKey != io.m_Parameters.end())
+        {
+            parameter = itKey->second;
+            return true;
+        }
+        return false;
+    };
+
+#define get_params(Param, Type, Typedecl, Default)                             \
+    Params.Param = Default;                                                    \
+    lf_Set##Type##Parameter(#Param, Params.Param);
+    BP5_FOREACH_PARAMETER_TYPE_4ARGS(get_params);
+#undef get_params
+};
+
+} // namespace engine
+} // namespace core
+} // namespace adios2

source/adios2/engine/bp5/BP5Engine.h

@@ -0,0 +1,141 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * BP5Writer.h
+ *
+ */
+
+#ifndef ADIOS2_ENGINE_BP5_BP5ENGINE_H_
+#define ADIOS2_ENGINE_BP5_BP5ENGINE_H_
+
+#include "adios2/common/ADIOSConfig.h"
+#include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
+#include "adios2/toolkit/burstbuffer/FileDrainerSingleThread.h"
+#include "adios2/toolkit/format/bp5/BP5Serializer.h"
+#include "adios2/toolkit/transportman/TransportMan.h"
+
+namespace adios2
+{
+namespace core
+{
+namespace engine
+{
+
+class BP5Engine
+{
+public:
+    int m_RankMPI = 0;
+    /* metadata index table*/
+    std::unordered_map<uint64_t, std::vector<uint64_t>> m_MetadataIndexTable;
+
+    struct Minifooter
+    {
+        std::string VersionTag;
+        uint64_t PGIndexStart = 0;
+        uint64_t VarsIndexStart = 0;
+        uint64_t AttributesIndexStart = 0;
+        int8_t Version = -1;
+        bool IsLittleEndian = true;
+        bool HasSubFiles = false;
+    };
+
+    format::BufferSTL m_MetadataIndex;
+
+    /** Positions of flags in Index Table Header that Reader uses */
+    static constexpr size_t m_IndexHeaderSize = 64;
+    static constexpr size_t m_EndianFlagPosition = 36;
+    static constexpr size_t m_BPVersionPosition = 37;
+    static constexpr size_t m_ActiveFlagPosition = 38;
+    static constexpr size_t m_BPMinorVersionPosition = 39;
+    static constexpr size_t m_WriterCountPosition = 40;
+    static constexpr size_t m_AggregatorCountPosition = 44;
+    static constexpr size_t m_ColumnMajorFlagPosition = 48;
+    static constexpr size_t m_VersionTagPosition = 0;
+    static constexpr size_t m_VersionTagLength = 32;
+
+    std::vector<std::string>
+    GetBPSubStreamNames(const std::vector<std::string> &names) const noexcept;
+
+    std::vector<std::string>
+    GetBPMetadataFileNames(const std::vector<std::string> &names) const
+        noexcept;
+    std::vector<std::string>
+    GetBPMetaMetadataFileNames(const std::vector<std::string> &names) const
+        noexcept;
+    std::string GetBPMetadataFileName(const std::string &name) const noexcept;
+    std::string GetBPMetaMetadataFileName(const std::string &name) const
+        noexcept;
+    std::vector<std::string>
+    GetBPMetadataIndexFileNames(const std::vector<std::string> &names) const
+        noexcept;
+
+    std::string GetBPMetadataIndexFileName(const std::string &name) const
+        noexcept;
+
+    std::string GetBPSubStreamName(const std::string &name, const size_t id,
+                                   const bool hasSubFiles = true,
+                                   const bool isReader = false) const noexcept;
+
+#define BP5_FOREACH_PARAMETER_TYPE_4ARGS(MACRO)                                \
+    MACRO(OpenTimeoutSecs, Int, int, 3600)                                     \
+    MACRO(BeginStepPollingFrequencySecs, Int, int, 0)                          \
+    MACRO(StreamReader, Bool, bool, false)                                     \
+    MACRO(BurstBufferDrain, Bool, bool, true)                                  \
+    MACRO(NodeLocal, Bool, bool, false)                                        \
+    MACRO(BurstBufferPath, String, std::string, "\"\"")                        \
+    MACRO(verbose, Int, int, 0)                                                \
+    MACRO(CollectiveMetadata, Bool, bool, true)                                \
+    MACRO(ReaderShortCircuitReads, Bool, bool, false)
+
+    struct BP5Params
+    {
+#define declare_struct(Param, Type, Typedecl, Default) Typedecl Param;
+        BP5_FOREACH_PARAMETER_TYPE_4ARGS(declare_struct)
+#undef declare_struct
+    };
+
+    void ParseParams(IO &io, BP5Params &Params);
+    BP5Params m_Parameters;
+
+private:
+};
+
+} // namespace engine
+} // namespace core
+} // namespace adios2
+#endif
+
+/*
+ *   Data Formats:
+ *   MetadataIndex file (md.idx)
+ *	BP5 header for "Index Table" (64 bytes)
+ *      for each Writer, what aggregator writes its data
+ *             uint16_t * WriterCount;
+ *	for each timestep:
+ *		uint64_t 0 :  CombinedMetaDataPos
+ *		uint64_t 1 :  CombinedMetaDataSize
+ *		for each Writer
+ *		 uint64_t  DataPos (in the file above)
+ *
+ *   MetaMetadata file (mmd.0) contains FFS format information
+ *	for each meta metadata item:
+ *		uint64_t  MetaMetaIDLen
+ *		uint64_t  MetaMetaInfoLen
+ *		char[MeatMetaIDLen] MetaMetaID
+ *		char[MetaMetaInfoLen] MetaMetanfo
+ *  Notes:   This file should be quite small, with size dependent upon the
+ *number of different "formats" written by any rank.
+ *
+ *
+ *   MetaData file (md.0) contains encoded metadata for each timestep, for each
+ *rank BP5 header for "Metadata" (64 bytes) for each timestep: uint64_t
+ *TotalSize of this metadata block (including this length) uint64_t[WriterCount]
+ *Length of each writer rank's metadata for each rank FFS-encoded metadata block
+ *of length corresponding to entry above
+ *
+ *
+ *   Data file (data.x) contains a block of data for each timestep, for each
+ *rank
+ */