Merge pull request #2806 from JasonRuonanWang/dataman

DataMan: Fix a few bugs for openPMD
ornladios · Jul 30, 2021 · 3609408 · 3609408
2 parents 779310a + 276ab62
commit 3609408
Show file tree

Hide file tree

Showing 4 changed files with 135 additions and 1 deletion.
diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp
@@ -169,6 +169,8 @@ StepStatus DataManReader::BeginStep(StepMode stepMode,
 
     m_Serializer.GetAttributes(m_IO);
 
+    m_IO.RemoveAllVariables();
+
     for (const auto &i : *m_CurrentStepMetadata)
     {
         if (i.step == static_cast<size_t>(m_CurrentStep))

diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -670,5 +670,127 @@ void DataManSerializer::Log(const int level, const std::string &message,
     }
 }
 
+template <>
+void DataManSerializer::PutData(
+    const std::string *inputData, const std::string &varName,
+    const Dims &varShape, const Dims &varStart, const Dims &varCount,
+    const Dims &varMemStart, const Dims &varMemCount, const std::string &doid,
+    const size_t step, const int rank, const std::string &address,
+    const std::vector<core::VariableBase::Operation> &ops, VecPtr localBuffer,
+    JsonPtr metadataJson)
+{
+    PERFSTUBS_SCOPED_TIMER_FUNC();
+    Log(1,
+        "DataManSerializer::PutData begin with Step " + std::to_string(step) +
+            " Var " + varName,
+        true, true);
+
+    if (localBuffer == nullptr)
+    {
+        localBuffer = m_LocalBuffer;
+    }
+
+    nlohmann::json metaj;
+
+    metaj["N"] = varName;
+    metaj["O"] = varStart;
+    metaj["C"] = varCount;
+    metaj["S"] = varShape;
+    metaj["Y"] = "string";
+    metaj["P"] = localBuffer->size();
+
+    if (not address.empty())
+    {
+        metaj["A"] = address;
+    }
+
+    if (not m_IsRowMajor)
+    {
+        metaj["M"] = m_IsRowMajor;
+    }
+    if (not m_IsLittleEndian)
+    {
+        metaj["E"] = m_IsLittleEndian;
+    }
+
+    metaj["I"] = inputData->size();
+
+    if (localBuffer->capacity() < localBuffer->size() + inputData->size())
+    {
+        localBuffer->reserve((localBuffer->size() + inputData->size()) * 2);
+    }
+
+    localBuffer->resize(localBuffer->size() + inputData->size());
+
+    std::memcpy(localBuffer->data() + localBuffer->size() - inputData->size(),
+                inputData->data(), inputData->size());
+
+    if (metadataJson == nullptr)
+    {
+        m_MetadataJson[std::to_string(step)][std::to_string(rank)].emplace_back(
+            std::move(metaj));
+    }
+    else
+    {
+        (*metadataJson)[std::to_string(step)][std::to_string(rank)]
+            .emplace_back(std::move(metaj));
+    }
+
+    Log(1,
+        "DataManSerializer::PutData end with Step " + std::to_string(step) +
+            " Var " + varName,
+        true, true);
+}
+
+template <>
+int DataManSerializer::GetData(std::string *outputData,
+                               const std::string &varName, const Dims &varStart,
+                               const Dims &varCount, const size_t step,
+                               const Dims &varMemStart, const Dims &varMemCount)
+{
+    PERFSTUBS_SCOPED_TIMER_FUNC();
+
+    DmvVecPtr vec = nullptr;
+
+    {
+        std::lock_guard<std::mutex> l(m_DataManVarMapMutex);
+        const auto &i = m_DataManVarMap.find(step);
+        if (i == m_DataManVarMap.end())
+        {
+            return -1; // step not found
+        }
+        else
+        {
+            vec = i->second;
+        }
+    }
+
+    if (vec == nullptr)
+    {
+        return -2; // step found but variable not found
+    }
+
+    char *input_data = nullptr;
+
+    for (const auto &j : *vec)
+    {
+        if (j.name == varName)
+        {
+            if (j.buffer == nullptr)
+            {
+                continue;
+            }
+            else
+            {
+                input_data = reinterpret_cast<char *>(j.buffer->data());
+            }
+
+            input_data += j.position;
+
+            *outputData = std::string(input_data, j.size);
+        }
+    }
+    return 0;
+}
 } // namespace format
 } // namespace adios2
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc
@@ -235,6 +235,7 @@ void DataManSerializer::PutData(
         datasize = std::accumulate(varCount.begin(), varCount.end(), sizeof(T),
                                    std::multiplies<size_t>());
     }
+
     metaj["I"] = datasize;
 
     if (localBuffer->capacity() < localBuffer->size() + datasize)

diff --git a/testing/adios2/engine/dataman/TestDataMan1D.cpp b/testing/adios2/engine/dataman/TestDataMan1D.cpp
@@ -123,6 +123,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
     auto bpDComplexes = dataManIO.DefineVariable<std::complex<double>>(
         "bpDComplexes", shape, start, count);
     auto bpUInt64s = dataManIO.DefineVariable<uint64_t>("bpUInt64s");
+    auto scalarString = dataManIO.DefineVariable<std::string>("scalarString");
     dataManIO.DefineAttribute<int>("AttInt", 110);
     adios2::Engine dataManWriter =
         dataManIO.Open("stream", adios2::Mode::Write);
@@ -151,6 +152,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
         dataManWriter.Put(bpDComplexes, myDComplexes.data(),
                           adios2::Mode::Sync);
         dataManWriter.Put(bpUInt64s, i);
+        dataManWriter.Put(scalarString, std::string("some text"));
         dataManWriter.EndStep();
     }
     dataManWriter.Close();
@@ -186,7 +188,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
         {
             received_steps = true;
             const auto &vars = dataManIO.AvailableVariables();
-            ASSERT_EQ(vars.size(), 11);
+            ASSERT_EQ(vars.size(), 12);
             currentStep = dataManReader.CurrentStep();
             adios2::Variable<char> bpChars =
                 dataManIO.InquireVariable<char>("bpChars");
@@ -210,6 +212,8 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
                 dataManIO.InquireVariable<std::complex<double>>("bpDComplexes");
             adios2::Variable<uint64_t> bpUInt64s =
                 dataManIO.InquireVariable<uint64_t>("bpUInt64s");
+            adios2::Variable<std::string> scalarString =
+                dataManIO.InquireVariable<std::string>("scalarString");
             auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars);
             bpChars.SetSelection({start, count});
             bpUChars.SetSelection({start, count});
@@ -233,6 +237,11 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
                               adios2::Mode::Sync);
             dataManReader.Get(bpDComplexes, myDComplexes.data(),
                               adios2::Mode::Sync);
+
+            std::string readString;
+            dataManReader.Get(scalarString, readString, adios2::Mode::Sync);
+            ASSERT_EQ(readString, "some text");
+
             uint64_t stepValue;
             dataManReader.Get(bpUInt64s, &stepValue, adios2::Mode::Sync);
             ASSERT_EQ(currentStep, stepValue);