diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 95d7cf223c..e0bd3edc85 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -849,7 +849,7 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) } case IndexRecord::StepRecord: { - m_AppendMetadataIndexPos = position; + m_AppendMetadataIndexPos = position - 9; const uint64_t MetadataPos = helper::ReadValue(buffer, position, IsLittleEndian); position += sizeof(uint64_t); // MetadataSize diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 02f7241d37..fcd1b1a514 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -125,6 +125,9 @@ if(ADIOS2_HAVE_BP5) gtest_add_tests_helper(ParameterSelectSteps MPI_ALLOW BP Engine.BP. .BP5 WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" ) + gtest_add_tests_helper(ParameterAppendAfterSteps MPI_ALLOW BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" + ) gtest_add_tests_helper(DirectIO MPI_NONE BP Engine.BP. .BP5 WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" ) diff --git a/testing/adios2/engine/bp/TestBPParameterAppendAfterSteps.cpp b/testing/adios2/engine/bp/TestBPParameterAppendAfterSteps.cpp new file mode 100644 index 0000000000..06fbad447f --- /dev/null +++ b/testing/adios2/engine/bp/TestBPParameterAppendAfterSteps.cpp @@ -0,0 +1,201 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Test "AppendAfterSteps" parameter for appending new steps to an existing BP + * file + * + * Created on: May 9, 2022 + * Author: Norbert Podhorszki + */ + +#include +#include + +#include +#include + +#include + +#include + +#include "../SmallTestData.h" + +std::string engineName; // comes from command line +const std::size_t Nx = 10; +using DataArray = std::array; + +class BPParameterAppendAfterSteps : public ::testing::Test +{ +public: + BPParameterAppendAfterSteps() = default; + + DataArray GenerateData(int step, int rank, int size) + { + DataArray d; + int j = rank + 1 + step * size; + for (size_t i = 0; i < d.size(); ++i) + { + d[i] = j; + } + return d; + } + + std::string ArrayToString(int32_t *data, size_t nelems) + { + std::stringstream ss; + ss << "["; + for (size_t i = 0; i < nelems; ++i) + { + ss << data[i]; + if (i < nelems - 1) + { + ss << " "; + } + } + ss << "]"; + return ss.str(); + } +}; + +class BPParameterAppendAfterStepsP +: public BPParameterAppendAfterSteps, + public ::testing::WithParamInterface> +{ +protected: + int GetNsteps1() { return std::get<0>(GetParam()); }; + int GetAppendAfterSteps() { return std::get<1>(GetParam()); }; + int GetNsteps2() { return std::get<2>(GetParam()); }; +}; + +TEST_P(BPParameterAppendAfterStepsP, Read) +{ + int mpiRank = 0, mpiSize = 1; + int nSteps1 = GetNsteps1(); + int nAppendAfterSteps = GetAppendAfterSteps(); + int nSteps2 = GetNsteps2(); +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + + std::string filename = + "ParameterAppendAfterSteps" + std::to_string(mpiSize) + "_" + + std::to_string(nSteps1) + "_" + std::to_string(nAppendAfterSteps) + + "_" + std::to_string(nSteps2) + ".bp"; + + const std::size_t Nx = 10; + adios2::Dims shape{static_cast(mpiSize * Nx)}; + adios2::Dims start{static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Nx)}; + + adios2::IO ioWrite = adios.DeclareIO("TestIOWrite"); + ioWrite.SetEngine(engineName); + auto var0 = ioWrite.DefineVariable("var", shape, start, count); + + /* + * Create file with nSteps1 + */ + + adios2::Engine engine = ioWrite.Open(filename, adios2::Mode::Write); + for (int step = 0; step < nSteps1; ++step) + { + auto d = GenerateData(step, mpiRank, mpiSize); + engine.BeginStep(); + engine.Put(var0, d.data()); + engine.EndStep(); + } + engine.Close(); + + /* + * Append nSteps2 after nAppendAfterSteps + */ + + int nStartStep = nAppendAfterSteps; + if (nAppendAfterSteps < 0) + { + nStartStep = nSteps1 + nAppendAfterSteps + 1; + if (nStartStep < 0) + { + nStartStep = 0; + } + } + int nTotalSteps = nStartStep + nSteps2; + + ioWrite.SetParameter("AppendAfterSteps", std::to_string(nAppendAfterSteps)); + engine = ioWrite.Open(filename, adios2::Mode::Append); + for (int step = nStartStep; step < nStartStep + nSteps2; ++step) + { + auto d = GenerateData(step, mpiRank, mpiSize); + engine.BeginStep(); + engine.Put(var0, d.data()); + engine.EndStep(); + } + engine.Close(); + + /* + * Read back every step + */ + + adios2::IO ioRead = adios.DeclareIO("TestIORead"); + ioRead.SetEngine(engineName); + adios2::Engine engine_s = + ioRead.Open(filename, adios2::Mode::ReadRandomAccess); + EXPECT_TRUE(engine_s); + + const size_t nsteps = engine_s.Steps(); + EXPECT_EQ(nsteps, nTotalSteps); + + adios2::Variable var = ioRead.InquireVariable("var"); + + for (size_t step = 0; step < nsteps; step++) + { + var.SetStepSelection(adios2::Box(step, 1)); + std::vector res; + var.SetSelection({{Nx * mpiRank}, {Nx}}); + engine_s.Get(var, res, adios2::Mode::Sync); + int s = static_cast(step); + auto d = GenerateData(s, mpiRank, mpiSize); + EXPECT_EQ(res[0], d[0]); + } + + engine_s.Close(); +#if ADIOS2_USE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif +} + +INSTANTIATE_TEST_SUITE_P( + BPParameterAppendAfterSteps, BPParameterAppendAfterStepsP, + ::testing::Values(std::make_tuple(2, 1, 2), std::make_tuple(3, 0, 2), + std::make_tuple(2, 2, 2), std::make_tuple(2, -1, 2), + std::make_tuple(2, -2, 2), std::make_tuple(2, -99, 2))); + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + MPI_Init(nullptr, nullptr); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +}