From a0d458676f7e6c6296a3b4176c2f08004efe2905 Mon Sep 17 00:00:00 2001 From: Dmitry Ganyushin Date: Tue, 7 Mar 2023 11:43:51 -0500 Subject: [PATCH 01/11] Fix for the issue #3503 --- .../format/dataman/DataManSerializer.tcc | 24 +++-- testing/adios2/engine/dataman/CMakeLists.txt | 6 ++ .../adios2/engine/dataman/TestDataMan1D.py | 91 ++++++++++++++++++ .../adios2/engine/dataman/TestDataMan1xN.py | 92 +++++++++++++++++++ .../engine/dataman/TestDataManSingleValues.py | 89 ++++++++++++++++++ 5 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 testing/adios2/engine/dataman/TestDataMan1D.py create mode 100644 testing/adios2/engine/dataman/TestDataMan1xN.py create mode 100644 testing/adios2/engine/dataman/TestDataManSingleValues.py diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 884c8e451b..e6667f05b8 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -256,11 +256,17 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, { input_data += j.position; } - - if (j.shape.size() > 0 and j.shape[0] > 1 and j.start.size() > 0 and - j.start.size() == j.count.size() and - j.start.size() == varStart.size() and - j.start.size() == varCount.size()) + /* single values */ + if (j.shape.empty() or + std::all_of(j.shape.begin(), j.shape.end(), + [&](size_t i) { return i == 1; })) + { + std::memcpy(reinterpret_cast(outputData), input_data, + sizeof(T)); + } + else if (j.start.size() > 0 and j.start.size() == j.count.size() and + j.start.size() == varStart.size() and + j.start.size() == varCount.size()) { if (m_ContiguousMajor) { @@ -279,10 +285,12 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, sizeof(T), j.start, j.count, varMemStart, varMemCount); } } - if (j.shape.empty() or (j.shape.size() == 1 and j.shape[0] == 1)) + else { - std::memcpy(reinterpret_cast(outputData), input_data, - sizeof(T)); + throw std::runtime_error( + "DataManSerializer::GeData end with Step \" + " + "std::to_string(step) +\n" + " \" Var \" + varName failed"); } } } diff --git a/testing/adios2/engine/dataman/CMakeLists.txt b/testing/adios2/engine/dataman/CMakeLists.txt index f40d6b60a6..812649cc21 100644 --- a/testing/adios2/engine/dataman/CMakeLists.txt +++ b/testing/adios2/engine/dataman/CMakeLists.txt @@ -17,6 +17,12 @@ foreach(tst IN ITEMS ) endforeach() +if (ADIOS2_HAVE_Python) + python_add_test(NAME Test.Engine.DataMan1D.Serial SCRIPT TestDataMan1D.py) + python_add_test(NAME Test.Engine.DataMan1xN.Serial SCRIPT TestDataMan1xN.py) + python_add_test(NAME Test.Engine.DataManSingleValues SCRIPT TestDataManSingleValues.py) +endif() + if(ADIOS2_HAVE_ZFP) gtest_add_tests_helper(2DZfp MPI_NONE DataMan Engine.DataMan. "") set_tests_properties(${Test.Engine.DataMan.2DZfp-TESTS} diff --git a/testing/adios2/engine/dataman/TestDataMan1D.py b/testing/adios2/engine/dataman/TestDataMan1D.py new file mode 100644 index 0000000000..8b8aac928f --- /dev/null +++ b/testing/adios2/engine/dataman/TestDataMan1D.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestDataMan1D.py: test for 1D data transfer by reading in Python +# Created on: March 3, 2023 +# Author: Dmitry Ganyushin ganyushindi@ornl.gov +from multiprocessing import Process +import unittest +import numpy as np +import adios2 + + +class TestDataMan1D(unittest.TestCase): + + def setUp(self): + self.conf = { + "IPAddress": "127.0.0.1", + "Port": "12306", + "Timeout": "5", + "TransportMode": "reliable", + "RendezvousReaderCount": "1", + } + self.Nx = 10 + self.fill_value = 1.0 + self.shape = [self.Nx] + + def test_run(self): + + s = Process(target=self.thread_send) + r = Process(target=self.thread_receive) + + s.start() + r.start() + + r.join() + s.join() + + def thread_send(self): + data = np.full(shape=self.shape, fill_value=self.fill_value) + shape = data.shape + count = shape + start = (0,) * len(shape) + + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Server") + wan.SetEngine("Dataman") + + wan.SetParameters(self.conf) + writer = wan.Open("testdata", adios2.Mode.Write) + sendbuffer = wan.DefineVariable("np_data", data, shape, + start, count, adios2.ConstantDims) + self.assertIsNotNone(sendbuffer) + if sendbuffer: + writer.BeginStep() + writer.Put(sendbuffer, data, adios2.Mode.Deferred) + writer.EndStep() + else: + raise ValueError("DefineVariable failed") + + writer.Close() + + def thread_receive(self): + data = np.zeros(shape=self.shape) + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Client") + wan.SetEngine("Dataman") + wan.SetParameters(self.conf) + reader = wan.Open("testdata", adios2.Mode.Read) + while True: + stepStatus = reader.BeginStep() + if stepStatus == adios2.StepStatus.OK: + recvar = wan.InquireVariable("np_data") + self.assertIsNotNone(recvar) + bufshape = recvar.Shape() + self.assertTrue(bufshape[0] == self.Nx) + reader.Get(recvar, data, adios2.Mode.Sync) + + elif stepStatus == adios2.StepStatus.EndOfStream: + break + else: + raise StopIteration() + reader.EndStep() + reader.Close() + self.assertTrue(all([data[i] == self.fill_value for i + in range(len(data))])) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/adios2/engine/dataman/TestDataMan1xN.py b/testing/adios2/engine/dataman/TestDataMan1xN.py new file mode 100644 index 0000000000..7097c798c7 --- /dev/null +++ b/testing/adios2/engine/dataman/TestDataMan1xN.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestDataMan1D.py: test for 1D data transfer by reading in Python +# Created on: March 3, 2023 +# Author: Dmitry Ganyushin ganyushindi@ornl.gov +from multiprocessing import Process +import unittest +import numpy as np +import adios2 + + +class TestDataMan1D(unittest.TestCase): + + def setUp(self): + self.conf = { + "IPAddress": "127.0.0.1", + "Port": "12306", + "Timeout": "5", + "TransportMode": "reliable", + "RendezvousReaderCount": "1", + } + self.Nx = 10 + self.fill_value = 1.0 + self.shape = [1, self.Nx] + + def test_run(self): + + s = Process(target=self.thread_send) + r = Process(target=self.thread_receive) + + s.start() + r.start() + + r.join() + s.join() + + def thread_send(self): + data = np.full(shape=self.shape, fill_value=self.fill_value) + shape = data.shape + count = shape + start = (0,) * len(shape) + + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Server") + wan.SetEngine("Dataman") + + wan.SetParameters(self.conf) + writer = wan.Open("testdata", adios2.Mode.Write) + sendbuffer = wan.DefineVariable("np_data", data, shape, + start, count, adios2.ConstantDims) + self.assertIsNotNone(sendbuffer) + if sendbuffer: + writer.BeginStep() + writer.Put(sendbuffer, data, adios2.Mode.Deferred) + writer.EndStep() + else: + raise ValueError("DefineVariable failed") + + writer.Close() + + def thread_receive(self): + data = np.zeros(shape=self.shape) + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Client") + wan.SetEngine("Dataman") + wan.SetParameters(self.conf) + reader = wan.Open("testdata", adios2.Mode.Read) + while True: + stepStatus = reader.BeginStep() + if stepStatus == adios2.StepStatus.OK: + recvar = wan.InquireVariable("np_data") + self.assertIsNotNone(recvar) + bufshape = recvar.Shape() + self.assertTrue(bufshape[0] == 1) + self.assertTrue(bufshape[1] == self.Nx) + reader.Get(recvar, data, adios2.Mode.Sync) + + elif stepStatus == adios2.StepStatus.EndOfStream: + break + else: + raise StopIteration() + reader.EndStep() + reader.Close() + self.assertTrue(all([data[0][i] == self.fill_value for i + in range(len(data))])) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/adios2/engine/dataman/TestDataManSingleValues.py b/testing/adios2/engine/dataman/TestDataManSingleValues.py new file mode 100644 index 0000000000..72e116f2b5 --- /dev/null +++ b/testing/adios2/engine/dataman/TestDataManSingleValues.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestDataMan1D.py: test for 1D data transfer by reading in Python +# Created on: March 3, 2023 +# Author: Dmitry Ganyushin ganyushindi@ornl.gov +from multiprocessing import Process +import unittest +import numpy as np +import adios2 + + +class TestDataMan1D(unittest.TestCase): + + def setUp(self): + self.conf = { + "IPAddress": "127.0.0.1", + "Port": "12306", + "Timeout": "5", + "TransportMode": "reliable", + "RendezvousReaderCount": "1", + } + self.Nx = 1 + self.fill_value = 1.0 + self.shape = [self.Nx] + + def test_run(self): + + s = Process(target=self.thread_send) + r = Process(target=self.thread_receive) + + s.start() + r.start() + + r.join() + s.join() + + def thread_send(self): + data = np.full(shape=self.shape, fill_value=self.fill_value) + shape = data.shape + count = shape + start = (0,) * len(shape) + + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Server") + wan.SetEngine("Dataman") + + wan.SetParameters(self.conf) + writer = wan.Open("testdata", adios2.Mode.Write) + sendbuffer = wan.DefineVariable("np_data", data, shape, + start, count, adios2.ConstantDims) + self.assertIsNotNone(sendbuffer) + if sendbuffer: + writer.BeginStep() + writer.Put(sendbuffer, data, adios2.Mode.Deferred) + writer.EndStep() + else: + raise ValueError("DefineVariable failed") + + writer.Close() + + def thread_receive(self): + data = np.zeros(shape=self.shape) + adios_io = adios2.ADIOS() + wan = adios_io.DeclareIO("Client") + wan.SetEngine("Dataman") + wan.SetParameters(self.conf) + reader = wan.Open("testdata", adios2.Mode.Read) + while True: + stepStatus = reader.BeginStep() + if stepStatus == adios2.StepStatus.OK: + recvar = wan.InquireVariable("np_data") + self.assertIsNotNone(recvar) + reader.Get(recvar, data, adios2.Mode.Sync) + + elif stepStatus == adios2.StepStatus.EndOfStream: + break + else: + raise StopIteration() + reader.EndStep() + reader.Close() + self.assertTrue(all([data[i] == self.fill_value for i + in range(len(data))])) + + +if __name__ == '__main__': + unittest.main() From 3a5efc94872400aab9f734cc10ffe5b5de203945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:57:16 +0000 Subject: [PATCH 02/11] Bump cryptography from 38.0.2 to 39.0.1 in /docs Bumps [cryptography](https://github.com/pyca/cryptography) from 38.0.2 to 39.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/38.0.2...39.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] (cherry picked from commit 1e2d0584e853b77b21d98f14a0c31a90931d858d) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0880ee66ed..a4c9f56942 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -7,7 +7,7 @@ certifi==2022.12.7 cffi==1.15.1 charset-normalizer==2.1.1 colorama==0.4.6 -cryptography==38.0.2 +cryptography==39.0.1 docutils==0.17 funcparserlib==1.0.1 idna==3.4 From 832702380730c2bac2da712276aaacbf1ddd6f47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:57:20 +0000 Subject: [PATCH 03/11] Bump numpy from 1.21.6 to 1.22.0 in /docs Bumps [numpy](https://github.com/numpy/numpy) from 1.21.6 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.21.6...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] (cherry picked from commit e054144c1e89f8ec1fa1e086dba8c86732e0c4f6) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a4c9f56942..542cffd5aa 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -16,7 +16,7 @@ importlib-metadata==4.11.4 Jinja2==3.1.2 MarkupSafe==2.1.1 mpi4py==3.1.3 -numpy==1.21.6 +numpy==1.22.0 packaging==22.0 Pillow==9.4.0 pip==22.3.1 From 4f75c5ae3bac79767abc6c7bcc99a8051fb12bf5 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 15 Mar 2023 18:32:07 -0400 Subject: [PATCH 04/11] Bug fix in BP5 when used with MemorySelection and GPU buffers --- source/adios2/engine/bp5/BP5Writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 0b0eaea8bd..bec7794e55 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -1803,7 +1803,7 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync) variable.m_MemoryCount, sourceRowMajor, false, (char *)ptr, variable.m_MemoryStart, variable.m_Count, sourceRowMajor, false, ObjSize, helper::CoreDims(), helper::CoreDims(), helper::CoreDims(), - helper::CoreDims(), false /* safemode */, MemorySpace::Host); + helper::CoreDims(), false /* safemode */, variable.m_MemSpace); } else { From 8e213ee333726e7b472e49f955990407003c83f2 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 15 Mar 2023 18:32:33 -0400 Subject: [PATCH 05/11] Testing for SetMemorySelection with CUDA buffers --- .../adios2/engine/bp/TestBPWriteReadCuda.cpp | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp index c0497f2340..83689bbc8e 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp @@ -199,6 +199,107 @@ void CUDADetectMemSpace(const std::string mode) } } +void CUDAWriteReadMemorySelection() +{ + const std::string fname("BPWRCUSel1D.bp"); + const size_t Nx = 10; + const size_t NSteps = 2; + const size_t ghostCells = 1; + std::vector r32s(Nx + 2 * ghostCells); + std::iota(r32s.begin(), r32s.end(), .0f); + + adios2::ADIOS adios; + { + // cuda simulation buffer + float *gpuSimData = nullptr; + cudaMalloc(&gpuSimData, (Nx + 2 * ghostCells) * sizeof(float)); + cudaMemcpy(gpuSimData, r32s.data(), + (Nx + 2 * ghostCells) * sizeof(float), + cudaMemcpyHostToDevice); + + adios2::IO io = adios.DeclareIO("TestIO"); + io.SetEngine("BP5"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + const adios2::Dims shape{static_cast(Nx)}; + const adios2::Dims start{static_cast(0)}; + const adios2::Dims count{Nx}; + auto var_r32 = io.DefineVariable("r32", shape, start, count); + + const adios2::Dims memoryStart = {ghostCells}; + const adios2::Dims memoryCount = {Nx + 2 * ghostCells}; + var_r32.SetMemorySelection({memoryStart, memoryCount}); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + cuda_increment(Nx + 2 * ghostCells, 1, 0, gpuSimData, INCREMENT); + + bpWriter.BeginStep(); + var_r32.SetMemorySpace(adios2::MemorySpace::GPU); + bpWriter.Put(var_r32, gpuSimData); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + { + // remove ghost cells from the input vector when checking correctness + r32s.erase(r32s.begin(), r32s.begin() + ghostCells); + r32s.erase(r32s.end() - ghostCells, r32s.end()); + + adios2::IO io = adios.DeclareIO("ReadIO"); + io.SetEngine("BP5"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + unsigned int t = 0; + for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++t) + { + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], Nx); + + auto mmR32 = std::minmax_element(r32s.begin(), r32s.end()); + EXPECT_EQ(var_r32.Min() - (t + 1) * INCREMENT, *mmR32.first); + EXPECT_EQ(var_r32.Max() - (t + 1) * INCREMENT, *mmR32.second); + + std::vector r32o(Nx); + float *gpuSimData; + cudaMalloc(&gpuSimData, Nx * sizeof(float)); + var_r32.SetMemorySpace(adios2::MemorySpace::GPU); + bpReader.Get(var_r32, gpuSimData); + bpReader.EndStep(); + cudaMemcpy(r32o.data(), gpuSimData, Nx * sizeof(float), + cudaMemcpyDeviceToHost); + + // Remove INCREMENT from each element + std::transform(r32o.begin(), r32o.end(), r32o.begin(), + std::bind(std::minus(), std::placeholders::_1, + (t + 1) * INCREMENT)); + for (size_t i = 0; i < Nx; i++) + { + char msg[1 << 8] = {0}; + snprintf(msg, sizeof(msg), "t=%d i=%zu r32o=%f r32s=%f", t, i, + r32o[i], r32s[i]); + ASSERT_LT(std::abs(r32o[i] - r32s[i]), EPSILON) << msg; + } + } + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + void CUDAWriteReadMPI1D(const std::string mode) { const std::string fname("BPWRCU1D_" + mode + ".bp"); @@ -343,6 +444,7 @@ class BPWRCUDA : public ::testing::TestWithParam TEST_P(BPWRCUDA, ADIOS2BPWRCUDA1D) { CUDAWriteReadMPI1D(GetParam()); } TEST_P(BPWRCUDA, ADIOS2BPCUDADetect) { CUDADetectMemSpace(GetParam()); } TEST_P(BPWRCUDA, ADIOS2BPCUDAWrong) { CUDAWrongMemSpace(); } +TEST_P(BPWRCUDA, ADIOS2BPCUDAMemSel) { CUDAWriteReadMemorySelection(); } INSTANTIATE_TEST_SUITE_P(CudaRW, BPWRCUDA, ::testing::Values("deferred", "sync")); From cc5dce6750d39c905828a851493c3d39fdccf97e Mon Sep 17 00:00:00 2001 From: anagainaru Date: Fri, 17 Mar 2023 11:12:19 -0400 Subject: [PATCH 06/11] Do not call putOperationInBuffer if all blocks have 0 elements Co-authored-by: lizdulac Co-authored-by: Norbert Podhorszki --- source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc index 4f13df82e5..07af6e8a5f 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc @@ -118,7 +118,13 @@ inline void BP4Serializer::PutVariablePayload( } else { - PutOperationPayloadInBuffer(variable, blockInfo); + const bool isZeroCount = + std::all_of(blockInfo.Count.begin(), blockInfo.Count.end(), + [](const size_t i) { return i == 0; }); + if (!isZeroCount) + { + PutOperationPayloadInBuffer(variable, blockInfo); + } } /* Now we can update the varLength including payload size including the From 7acf7b9e29665b93c0c2535619050e5a54ff01d6 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sat, 18 Mar 2023 20:05:57 -0400 Subject: [PATCH 07/11] Throw an exception in Py bindings on Get buffer type mismatch --- bindings/Python/py11Engine.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp index 8d3bff8f3a..aece436f03 100644 --- a/bindings/Python/py11Engine.cpp +++ b/bindings/Python/py11Engine.cpp @@ -129,6 +129,12 @@ void Engine::Get(Variable variable, pybind11::array &array, const Mode launch) #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ + if (!array.dtype().is(pybind11::dtype::of())) \ + { \ + throw std::invalid_argument( \ + "In ADIOS2 Get - Type mismatch between Python buffer and " \ + "incoming data."); \ + } \ m_Engine->Get( \ *dynamic_cast *>(variable.m_VariableBase), \ reinterpret_cast(const_cast(array.data())), launch); \ From 740c5bffdcb6ec5284a6e49b93b12818f862e815 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sun, 19 Mar 2023 11:03:23 -0400 Subject: [PATCH 08/11] Add test for Py Get exceptions --- testing/adios2/bindings/python/CMakeLists.txt | 1 + .../bindings/python/TestGetException_nompi.py | 133 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 testing/adios2/bindings/python/TestGetException_nompi.py diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index e79ada6a71..fb5cf105b5 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -16,6 +16,7 @@ python_add_test(NAME Bindings.Python.HighLevelAPI.Serial SCRIPT TestHighLevelAPI python_add_test(NAME Bindings.Python.BPWriteReadTypes.Serial SCRIPT TestBPWriteReadTypes_nompi.py) python_add_test(NAME Bindings.Python.BPSelectSteps.Serial SCRIPT TestBPSelectSteps_nompi.py) +python_add_test(NAME Bindings.Python.TypeExceptionOnGet.Serial SCRIPT TestGetException_nompi.py) if(ADIOS2_HAVE_MPI) add_python_mpi_test(BPWriteReadTypes) diff --git a/testing/adios2/bindings/python/TestGetException_nompi.py b/testing/adios2/bindings/python/TestGetException_nompi.py new file mode 100644 index 0000000000..7c501c0626 --- /dev/null +++ b/testing/adios2/bindings/python/TestGetException_nompi.py @@ -0,0 +1,133 @@ + +import numpy as np +import logging +import adios2 + +if __name__ == '__main__': + __spec__ = None + + +def main(): + + print("====================================") + + format = "%(asctime)s: %(message)s" + logging.basicConfig(format=format, level=logging.INFO, + datefmt="%H:%M:%S") + writer() + reader_fail() + reader_success() + + +def writer(): + + logging.info(" Writer: initiating writing") + data = np.arange(3, dtype=np.float32) + logging.info(f" data dtypes: {data.dtype!s}") + shape = data.shape + count = shape + start = (0,)*len(shape) + logging.info(f" data on writer side {data!s}") + + adios_io = adios2.ADIOS() + IO = adios_io.DeclareIO("writer") + + writer = IO.Open("testdatafile", adios2.Mode.Write) + writebuffer = IO.DefineVariable("np_data", data, shape, start, + count, adios2.ConstantDims) + if writebuffer: + writer.BeginStep() + writer.Put(writebuffer, data, adios2.Mode.Sync) + writer.EndStep() + else: + raise ValueError("DefineVariable failed") + + writer.Close() + + logging.info(" Writer: writing finished") + + +def reader_fail(): + + adios_io = adios2.ADIOS() + io = adios_io.DeclareIO("reader") + + logging.info(" Reader: initiating erroneous reading ") + reader = io.Open("testdatafile", adios2.Mode.Read) + while True: + stepStatus = reader.BeginStep() + if stepStatus == adios2.StepStatus.OK: + # inquire for variable + recvar = io.InquireVariable("np_data") + if recvar: + got_exception = 0 + # determine the shape of the data that will be sent + bufshape = recvar.Shape() + typ = recvar.Type() + logging.info(" Incoming variable type is " + typ) + # allocate buffer for new numpy + data = np.ones(bufshape) + logging.info(f" receive buffer dtype: {data.dtype!s}") + try: + reader.Get(recvar, data, adios2.Mode.Sync) + except Exception as e: + got_exception = 1 + logging.info(str(e)) + if not got_exception: + raise Exception("Didn't get an exception as expected") + else: + logging.info(" Got expected exception") + else: + raise ValueError("InquireVariable failed") + elif stepStatus == adios2.StepStatus.EndOfStream: + break + else: + raise StopIteration(f"next step failed to initiate {stepStatus!s}") + reader.EndStep() + reader.Close() + logging.info(" Reader: finished reading",) + + +def reader_success(): + + adios_io = adios2.ADIOS() + io = adios_io.DeclareIO("reader") + + logging.info(" Reader: initiating successful reading ") + reader = io.Open("testdatafile", adios2.Mode.Read) + while True: + stepStatus = reader.BeginStep() + if stepStatus == adios2.StepStatus.OK: + # inquire for variable + recvar = io.InquireVariable("np_data") + if recvar: + got_exception = 0 + # determine the shape of the data that will be sent + bufshape = recvar.Shape() + typ = recvar.Type() + logging.info(" Incoming variable type is " + typ) + # allocate buffer for new numpy + data = np.ones(bufshape, dtype=np.float32) + logging.info(f" receive buffer dtype: {data.dtype!s}") + try: + reader.Get(recvar, data, adios2.Mode.Sync) + except Exception as e: + got_exception = 1 + logging.info(str(e)) + if got_exception: + raise Exception(" Got unexpected exception") + else: + logging.info(" No exception as expected") + else: + raise ValueError("InquireVariable failed") + elif stepStatus == adios2.StepStatus.EndOfStream: + break + else: + raise StopIteration(f"next step failed to initiate {stepStatus!s}") + reader.EndStep() + reader.Close() + logging.info(" Reader: finished reading",) + + +if __name__ == "__main__": + main() From ea11e7b2d550d827c3a9d7e41be0df467d2d812c Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sun, 19 Mar 2023 11:05:50 -0400 Subject: [PATCH 09/11] Format --- testing/adios2/bindings/python/TestGetException_nompi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/adios2/bindings/python/TestGetException_nompi.py b/testing/adios2/bindings/python/TestGetException_nompi.py index 7c501c0626..77248d92fa 100644 --- a/testing/adios2/bindings/python/TestGetException_nompi.py +++ b/testing/adios2/bindings/python/TestGetException_nompi.py @@ -26,7 +26,7 @@ def writer(): logging.info(f" data dtypes: {data.dtype!s}") shape = data.shape count = shape - start = (0,)*len(shape) + start = (0,) * len(shape) logging.info(f" data on writer side {data!s}") adios_io = adios2.ADIOS() From 2ea9808659e7665e9c6aa80bfddb071a051830c9 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Fri, 17 Mar 2023 16:04:11 -0400 Subject: [PATCH 10/11] Throw an exception if an unimplemented function is called in BP5 --- source/adios2/engine/bp5/BP5Reader.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 80034dedba..bae32f4f69 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -1251,6 +1251,9 @@ void BP5Reader::DoClose(const int transportIndex) std::vector::BPInfo> BP5Reader::DoBlocksInfo( \ const Variable &variable, const size_t step) const \ { \ + helper::Throw( \ + "Engine", "BP5Reader", "DoBlocksInfo", \ + "Internal DoBlocksInfo should not be called in BP5Reader"); \ return std::vector::BPInfo>(); \ } From 92236608666aff9340cf884da857cb958b7d12c3 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sat, 18 Mar 2023 07:10:39 -0400 Subject: [PATCH 11/11] Easier to kill function and rely upon default method (which throws an exception) --- source/adios2/engine/bp5/BP5Reader.cpp | 14 -------------- source/adios2/engine/bp5/BP5Reader.h | 7 ------- 2 files changed, 21 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index bae32f4f69..1e8b6abf00 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -1246,20 +1246,6 @@ void BP5Reader::DoClose(const int transportIndex) } } -// DoBlocksInfo will not be called because MinBlocksInfo is operative -#define declare_type(T) \ - std::vector::BPInfo> BP5Reader::DoBlocksInfo( \ - const Variable &variable, const size_t step) const \ - { \ - helper::Throw( \ - "Engine", "BP5Reader", "DoBlocksInfo", \ - "Internal DoBlocksInfo should not be called in BP5Reader"); \ - return std::vector::BPInfo>(); \ - } - -ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - size_t BP5Reader::DoSteps() const { return m_StepsCount; } void BP5Reader::NotifyEngineNoVarsQuery() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index b8cd74d4b3..d159ce83d5 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -216,13 +216,6 @@ class BP5Reader : public BP5Engine, public Engine template void ReadVariableBlocks(Variable &variable); -#define declare_type(T) \ - std::vector::BPInfo> DoBlocksInfo( \ - const Variable &variable, const size_t step) const final; - - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - size_t DoSteps() const final; void DoGetAbsoluteSteps(const VariableBase &variable,