Release 29

bindings/Python/py11Engine.cpp

@@ -129,6 +129,12 @@ void Engine::Get(Variable variable, pybind11::array &array, const Mode launch)
 #define declare_type(T)                                                        \
     else if (type == helper::GetDataType<T>())                                 \
     {                                                                          \
+        if (!array.dtype().is(pybind11::dtype::of<T>()))                       \
+        {                                                                      \
+            throw std::invalid_argument(                                       \
+                "In ADIOS2 Get - Type mismatch between Python buffer and "     \
+                "incoming data.");                                             \
+        }                                                                      \
         m_Engine->Get(                                                         \
             *dynamic_cast<core::Variable<T> *>(variable.m_VariableBase),       \
             reinterpret_cast<T *>(const_cast<void *>(array.data())), launch);  \

docs/requirements.txt

@@ -7,7 +7,7 @@ certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==2.1.1
 colorama==0.4.6
-cryptography==38.0.2
+cryptography==39.0.1
 docutils==0.17
 funcparserlib==1.0.1
 idna==3.4
@@ -16,7 +16,7 @@ importlib-metadata==4.11.4
 Jinja2==3.1.2
 MarkupSafe==2.1.1
 mpi4py==3.1.3
-numpy==1.21.6
+numpy==1.22.0
 packaging==22.0
 Pillow==9.4.0
 pip==22.3.1

source/adios2/engine/bp5/BP5Reader.cpp

@@ -1246,17 +1246,6 @@ void BP5Reader::DoClose(const int transportIndex)
     }
 }
 
-// DoBlocksInfo will not be called because MinBlocksInfo is operative
-#define declare_type(T)                                                        \
-    std::vector<typename Variable<T>::BPInfo> BP5Reader::DoBlocksInfo(         \
-        const Variable<T> &variable, const size_t step) const                  \
-    {                                                                          \
-        return std::vector<typename Variable<T>::BPInfo>();                    \
-    }
-
-ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
-
 size_t BP5Reader::DoSteps() const { return m_StepsCount; }
 
 void BP5Reader::NotifyEngineNoVarsQuery()

source/adios2/engine/bp5/BP5Reader.h

@@ -216,13 +216,6 @@ class BP5Reader : public BP5Engine, public Engine
     template <class T>
     void ReadVariableBlocks(Variable<T> &variable);
 
-#define declare_type(T)                                                        \
-    std::vector<typename Variable<T>::BPInfo> DoBlocksInfo(                    \
-        const Variable<T> &variable, const size_t step) const final;
-
-    ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
-#undef declare_type
-
     size_t DoSteps() const final;
 
     void DoGetAbsoluteSteps(const VariableBase &variable,

source/adios2/engine/bp5/BP5Writer.cpp

@@ -1803,7 +1803,7 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync)
             variable.m_MemoryCount, sourceRowMajor, false, (char *)ptr,
             variable.m_MemoryStart, variable.m_Count, sourceRowMajor, false,
             ObjSize, helper::CoreDims(), helper::CoreDims(), helper::CoreDims(),
-            helper::CoreDims(), false /* safemode */, MemorySpace::Host);
+            helper::CoreDims(), false /* safemode */, variable.m_MemSpace);
     }
     else
     {

source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc

@@ -118,7 +118,13 @@ inline void BP4Serializer::PutVariablePayload(
     }
     else
     {
-        PutOperationPayloadInBuffer(variable, blockInfo);
+        const bool isZeroCount =
+            std::all_of(blockInfo.Count.begin(), blockInfo.Count.end(),
+                        [](const size_t i) { return i == 0; });
+        if (!isZeroCount)
+        {
+            PutOperationPayloadInBuffer(variable, blockInfo);
+        }
     }
 
     /* Now we can update the varLength including payload size including the

source/adios2/toolkit/format/dataman/DataManSerializer.tcc

@@ -261,11 +261,17 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
             {
                 input_data += j.position;
             }
-
-            if (j.shape.size() > 0 and j.shape[0] > 1 and j.start.size() > 0 and
-                j.start.size() == j.count.size() and
-                j.start.size() == varStart.size() and
-                j.start.size() == varCount.size())
+            /* single values */
+            if (j.shape.empty() or
+                std::all_of(j.shape.begin(), j.shape.end(),
+                            [&](size_t i) { return i == 1; }))
+            {
+                std::memcpy(reinterpret_cast<char *>(outputData), input_data,
+                            sizeof(T));
+            }
+            else if (j.start.size() > 0 and j.start.size() == j.count.size() and
+                     j.start.size() == varStart.size() and
+                     j.start.size() == varCount.size())
             {
                 if (m_ContiguousMajor)
                 {
@@ -284,10 +290,12 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
                         sizeof(T), j.start, j.count, varMemStart, varMemCount);
                 }
             }
-            if (j.shape.empty() or (j.shape.size() == 1 and j.shape[0] == 1))
+            else
             {
-                std::memcpy(reinterpret_cast<char *>(outputData), input_data,
-                            sizeof(T));
+                throw std::runtime_error(
+                    "DataManSerializer::GeData end with Step \" + "
+                    "std::to_string(step) +\n"
+                    "                        \" Var \" + varName failed");
             }
         }
     }

testing/adios2/bindings/python/CMakeLists.txt

@@ -16,6 +16,7 @@ python_add_test(NAME Bindings.Python.HighLevelAPI.Serial SCRIPT TestHighLevelAPI
 
 python_add_test(NAME Bindings.Python.BPWriteReadTypes.Serial SCRIPT TestBPWriteReadTypes_nompi.py)
 python_add_test(NAME Bindings.Python.BPSelectSteps.Serial SCRIPT TestBPSelectSteps_nompi.py)
+python_add_test(NAME Bindings.Python.TypeExceptionOnGet.Serial SCRIPT TestGetException_nompi.py)
 
 if(ADIOS2_HAVE_MPI)
   add_python_mpi_test(BPWriteReadTypes)

testing/adios2/bindings/python/TestGetException_nompi.py

@@ -0,0 +1,133 @@
+
+import numpy as np
+import logging
+import adios2
+
+if __name__ == '__main__':
+    __spec__ = None
+
+
+def main():
+
+    print("====================================")
+
+    format = "%(asctime)s: %(message)s"
+    logging.basicConfig(format=format, level=logging.INFO,
+                        datefmt="%H:%M:%S")
+    writer()
+    reader_fail()
+    reader_success()
+
+
+def writer():
+
+    logging.info(" Writer: initiating writing")
+    data = np.arange(3, dtype=np.float32)
+    logging.info(f"        data dtypes: {data.dtype!s}")
+    shape = data.shape
+    count = shape
+    start = (0,) * len(shape)
+    logging.info(f"        data on writer side {data!s}")
+
+    adios_io = adios2.ADIOS()
+    IO = adios_io.DeclareIO("writer")
+
+    writer = IO.Open("testdatafile", adios2.Mode.Write)
+    writebuffer = IO.DefineVariable("np_data", data, shape, start,
+                                    count, adios2.ConstantDims)
+    if writebuffer:
+        writer.BeginStep()
+        writer.Put(writebuffer, data, adios2.Mode.Sync)
+        writer.EndStep()
+    else:
+        raise ValueError("DefineVariable failed")
+
+    writer.Close()
+
+    logging.info(" Writer: writing finished")
+
+
+def reader_fail():
+
+    adios_io = adios2.ADIOS()
+    io = adios_io.DeclareIO("reader")
+
+    logging.info(" Reader: initiating erroneous reading ")
+    reader = io.Open("testdatafile", adios2.Mode.Read)
+    while True:
+        stepStatus = reader.BeginStep()
+        if stepStatus == adios2.StepStatus.OK:
+            # inquire for variable
+            recvar = io.InquireVariable("np_data")
+            if recvar:
+                got_exception = 0
+                # determine the shape of the data that will be sent
+                bufshape = recvar.Shape()
+                typ = recvar.Type()
+                logging.info("        Incoming variable type is " + typ)
+                # allocate buffer for new numpy
+                data = np.ones(bufshape)
+                logging.info(f"        receive buffer  dtype: {data.dtype!s}")
+                try:
+                    reader.Get(recvar, data, adios2.Mode.Sync)
+                except Exception as e:
+                    got_exception = 1
+                    logging.info(str(e))
+                if not got_exception:
+                    raise Exception("Didn't get an exception as expected")
+                else:
+                    logging.info("        Got expected exception")
+            else:
+                raise ValueError("InquireVariable failed")
+        elif stepStatus == adios2.StepStatus.EndOfStream:
+            break
+        else:
+            raise StopIteration(f"next step failed to initiate {stepStatus!s}")
+        reader.EndStep()
+    reader.Close()
+    logging.info(" Reader: finished reading",)
+
+
+def reader_success():
+
+    adios_io = adios2.ADIOS()
+    io = adios_io.DeclareIO("reader")
+
+    logging.info(" Reader: initiating successful reading ")
+    reader = io.Open("testdatafile", adios2.Mode.Read)
+    while True:
+        stepStatus = reader.BeginStep()
+        if stepStatus == adios2.StepStatus.OK:
+            # inquire for variable
+            recvar = io.InquireVariable("np_data")
+            if recvar:
+                got_exception = 0
+                # determine the shape of the data that will be sent
+                bufshape = recvar.Shape()
+                typ = recvar.Type()
+                logging.info("        Incoming variable type is " + typ)
+                # allocate buffer for new numpy
+                data = np.ones(bufshape, dtype=np.float32)
+                logging.info(f"        receive buffer  dtype: {data.dtype!s}")
+                try:
+                    reader.Get(recvar, data, adios2.Mode.Sync)
+                except Exception as e:
+                    got_exception = 1
+                    logging.info(str(e))
+                if got_exception:
+                    raise Exception("        Got unexpected exception")
+                else:
+                    logging.info("        No exception as expected")
+            else:
+                raise ValueError("InquireVariable failed")
+        elif stepStatus == adios2.StepStatus.EndOfStream:
+            break
+        else:
+            raise StopIteration(f"next step failed to initiate {stepStatus!s}")
+        reader.EndStep()
+    reader.Close()
+    logging.info(" Reader: finished reading",)
+
+
+if __name__ == "__main__":
+    main()

testing/adios2/engine/bp/TestBPWriteReadCuda.cpp

@@ -199,6 +199,107 @@ void CUDADetectMemSpace(const std::string mode)
     }
 }
 
+void CUDAWriteReadMemorySelection()
+{
+    const std::string fname("BPWRCUSel1D.bp");
+    const size_t Nx = 10;
+    const size_t NSteps = 2;
+    const size_t ghostCells = 1;
+    std::vector<float> r32s(Nx + 2 * ghostCells);
+    std::iota(r32s.begin(), r32s.end(), .0f);
+
+    adios2::ADIOS adios;
+    {
+        // cuda simulation buffer
+        float *gpuSimData = nullptr;
+        cudaMalloc(&gpuSimData, (Nx + 2 * ghostCells) * sizeof(float));
+        cudaMemcpy(gpuSimData, r32s.data(),
+                   (Nx + 2 * ghostCells) * sizeof(float),
+                   cudaMemcpyHostToDevice);
+
+        adios2::IO io = adios.DeclareIO("TestIO");
+        io.SetEngine("BP5");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        const adios2::Dims shape{static_cast<size_t>(Nx)};
+        const adios2::Dims start{static_cast<size_t>(0)};
+        const adios2::Dims count{Nx};
+        auto var_r32 = io.DefineVariable<float>("r32", shape, start, count);
+
+        const adios2::Dims memoryStart = {ghostCells};
+        const adios2::Dims memoryCount = {Nx + 2 * ghostCells};
+        var_r32.SetMemorySelection({memoryStart, memoryCount});
+
+        adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
+
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            cuda_increment(Nx + 2 * ghostCells, 1, 0, gpuSimData, INCREMENT);
+
+            bpWriter.BeginStep();
+            var_r32.SetMemorySpace(adios2::MemorySpace::GPU);
+            bpWriter.Put(var_r32, gpuSimData);
+            bpWriter.EndStep();
+        }
+
+        bpWriter.Close();
+    }
+    {
+        // remove ghost cells from the input vector when checking correctness
+        r32s.erase(r32s.begin(), r32s.begin() + ghostCells);
+        r32s.erase(r32s.end() - ghostCells, r32s.end());
+
+        adios2::IO io = adios.DeclareIO("ReadIO");
+        io.SetEngine("BP5");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);
+
+        unsigned int t = 0;
+        for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++t)
+        {
+            auto var_r32 = io.InquireVariable<float>("r32");
+            EXPECT_TRUE(var_r32);
+            ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray);
+            ASSERT_EQ(var_r32.Shape()[0], Nx);
+
+            auto mmR32 = std::minmax_element(r32s.begin(), r32s.end());
+            EXPECT_EQ(var_r32.Min() - (t + 1) * INCREMENT, *mmR32.first);
+            EXPECT_EQ(var_r32.Max() - (t + 1) * INCREMENT, *mmR32.second);
+
+            std::vector<float> r32o(Nx);
+            float *gpuSimData;
+            cudaMalloc(&gpuSimData, Nx * sizeof(float));
+            var_r32.SetMemorySpace(adios2::MemorySpace::GPU);
+            bpReader.Get(var_r32, gpuSimData);
+            bpReader.EndStep();
+            cudaMemcpy(r32o.data(), gpuSimData, Nx * sizeof(float),
+                       cudaMemcpyDeviceToHost);
+
+            // Remove INCREMENT from each element
+            std::transform(r32o.begin(), r32o.end(), r32o.begin(),
+                           std::bind(std::minus<float>(), std::placeholders::_1,
+                                     (t + 1) * INCREMENT));
+            for (size_t i = 0; i < Nx; i++)
+            {
+                char msg[1 << 8] = {0};
+                snprintf(msg, sizeof(msg), "t=%d i=%zu r32o=%f r32s=%f", t, i,
+                         r32o[i], r32s[i]);
+                ASSERT_LT(std::abs(r32o[i] - r32s[i]), EPSILON) << msg;
+            }
+        }
+        EXPECT_EQ(t, NSteps);
+
+        bpReader.Close();
+    }
+}
+
 void CUDAWriteReadMPI1D(const std::string mode)
 {
     const std::string fname("BPWRCU1D_" + mode + ".bp");
@@ -343,6 +444,7 @@ class BPWRCUDA : public ::testing::TestWithParam<std::string>
 TEST_P(BPWRCUDA, ADIOS2BPWRCUDA1D) { CUDAWriteReadMPI1D(GetParam()); }
 TEST_P(BPWRCUDA, ADIOS2BPCUDADetect) { CUDADetectMemSpace(GetParam()); }
 TEST_P(BPWRCUDA, ADIOS2BPCUDAWrong) { CUDAWrongMemSpace(); }
+TEST_P(BPWRCUDA, ADIOS2BPCUDAMemSel) { CUDAWriteReadMemorySelection(); }
 
 INSTANTIATE_TEST_SUITE_P(CudaRW, BPWRCUDA,
                          ::testing::Values("deferred", "sync"));

testing/adios2/engine/dataman/CMakeLists.txt

@@ -17,6 +17,12 @@ foreach(tst IN ITEMS
   )
 endforeach()
 
+if (ADIOS2_HAVE_Python)
+    python_add_test(NAME Test.Engine.DataMan1D.Serial SCRIPT TestDataMan1D.py)
+    python_add_test(NAME Test.Engine.DataMan1xN.Serial SCRIPT TestDataMan1xN.py)
+    python_add_test(NAME Test.Engine.DataManSingleValues SCRIPT TestDataManSingleValues.py)
+endif()
+
 if(ADIOS2_HAVE_ZFP)
     gtest_add_tests_helper(2DZfp MPI_NONE DataMan Engine.DataMan. "")
     set_tests_properties(${Test.Engine.DataMan.2DZfp-TESTS}