From 97ac1c741773d2d6b2ad5a550f1d986e394f2a03 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 14:36:35 -0400 Subject: [PATCH 1/9] mpi: get rid off ADIOSMPICommOnly.h It does not appear to be needed anymore. --- bindings/C/c/adios2_c_operator.h | 2 -- bindings/Python/py11ADIOS.h | 1 - bindings/Python/py11File.h | 1 - source/adios2/ADIOSMPICommOnly.h | 36 ------------------- source/adios2/CMakeLists.txt | 2 +- source/adios2/core/ADIOS.h | 2 +- source/adios2/core/Engine.h | 1 - source/adios2/core/IO.h | 1 - source/adios2/core/Stream.h | 1 - source/adios2/engine/hdf5/HDF5WriterP.h | 1 - source/adios2/toolkit/format/bp3/BP3Base.h | 1 - source/adios2/toolkit/format/bp4/BP4Base.h | 1 - .../adios2/toolkit/interop/hdf5/HDF5Common.h | 1 - source/adios2/toolkit/transport/Transport.h | 2 +- .../toolkit/transportman/TransportMan.h | 1 - source/utils/adios_reorganize/Reorganize.h | 1 - source/utils/bpls/bpls.h | 1 - 17 files changed, 3 insertions(+), 53 deletions(-) delete mode 100644 source/adios2/ADIOSMPICommOnly.h diff --git a/bindings/C/c/adios2_c_operator.h b/bindings/C/c/adios2_c_operator.h index 34bae3f032..b79e82ae1d 100644 --- a/bindings/C/c/adios2_c_operator.h +++ b/bindings/C/c/adios2_c_operator.h @@ -13,8 +13,6 @@ #include "adios2_c_types.h" -#include "adios2/ADIOSMPICommOnly.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/bindings/Python/py11ADIOS.h b/bindings/Python/py11ADIOS.h index ac8777d3be..0279fe6785 100644 --- a/bindings/Python/py11ADIOS.h +++ b/bindings/Python/py11ADIOS.h @@ -17,7 +17,6 @@ #include //std::shared_ptr #include -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/core/ADIOS.h" namespace adios2 diff --git a/bindings/Python/py11File.h b/bindings/Python/py11File.h index bebc566eaa..6d8bea0c12 100644 --- a/bindings/Python/py11File.h +++ b/bindings/Python/py11File.h @@ -13,7 +13,6 @@ #include -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/Stream.h" diff --git a/source/adios2/ADIOSMPICommOnly.h b/source/adios2/ADIOSMPICommOnly.h deleted file mode 100644 index bd066757a7..0000000000 --- a/source/adios2/ADIOSMPICommOnly.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ - -#ifndef ADIOS2_ADIOSMPICOMMONLY_H_ -#define ADIOS2_ADIOSMPICOMMONLY_H_ - -#include "adios2/ADIOSConfig.h" - -#ifdef ADIOS2_HAVE_MPI -#include -#else -#ifdef __cplusplus - -namespace adios2 -{ -namespace helper -{ -namespace mpi -{ -using MPI_Comm = int; -} // end namespace mpi -} // end namespace helper -} // end namespace adios2 -namespace adios2 -{ -using helper::mpi::MPI_Comm; -} -#else -typedef int MPI_Comm; -#endif - -#endif - -#endif /* ADIOS2_ADIOSMPICOMMONLY_H_ */ diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 88ddec4df3..d282636a27 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -223,7 +223,7 @@ set_target_properties(adios2 PROPERTIES SOVERSION ${ADIOS2_VERSION_MAJOR} ) -install(FILES ADIOSMacros.h ADIOSTypes.h ADIOSTypes.inl ADIOSMPICommOnly.h +install(FILES ADIOSMacros.h ADIOSTypes.h ADIOSTypes.inl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/adios2 ) diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index a4bb4d4676..b9687f560d 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -19,7 +19,7 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" +#include "adios2/ADIOSMPI.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/Operator.h" diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 0a8c02ca45..5c87db7f9a 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -24,7 +24,6 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/IO.h" diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 4caab055c2..662a3a41a9 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -21,7 +21,6 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/ADIOS.h" diff --git a/source/adios2/core/Stream.h b/source/adios2/core/Stream.h index e60dfc9761..43d1b3f19b 100644 --- a/source/adios2/core/Stream.h +++ b/source/adios2/core/Stream.h @@ -13,7 +13,6 @@ #include //std::shared_ptr -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/ADIOS.h" diff --git a/source/adios2/engine/hdf5/HDF5WriterP.h b/source/adios2/engine/hdf5/HDF5WriterP.h index a4a9c9f267..62cda8cf3c 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.h +++ b/source/adios2/engine/hdf5/HDF5WriterP.h @@ -14,7 +14,6 @@ #include #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/core/Engine.h" #include "adios2/core/IO.h" #include "adios2/toolkit/interop/hdf5/HDF5Common.h" diff --git a/source/adios2/toolkit/format/bp3/BP3Base.h b/source/adios2/toolkit/format/bp3/BP3Base.h index 2f32dc02b2..7322474b82 100644 --- a/source/adios2/toolkit/format/bp3/BP3Base.h +++ b/source/adios2/toolkit/format/bp3/BP3Base.h @@ -23,7 +23,6 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/Engine.h" diff --git a/source/adios2/toolkit/format/bp4/BP4Base.h b/source/adios2/toolkit/format/bp4/BP4Base.h index bc60e715d9..bc411dcf72 100644 --- a/source/adios2/toolkit/format/bp4/BP4Base.h +++ b/source/adios2/toolkit/format/bp4/BP4Base.h @@ -23,7 +23,6 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/Engine.h" diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h index f1835ac064..2bb246a726 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h @@ -15,7 +15,6 @@ #include -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/ADIOSTypes.h" #include "adios2/core/IO.h" // for CreateVar diff --git a/source/adios2/toolkit/transport/Transport.h b/source/adios2/toolkit/transport/Transport.h index 7c1a0f5233..1da6b7e742 100644 --- a/source/adios2/toolkit/transport/Transport.h +++ b/source/adios2/toolkit/transport/Transport.h @@ -17,7 +17,7 @@ /// \endcond #include "adios2/ADIOSConfig.h" -#include "adios2/ADIOSMPICommOnly.h" +#include "adios2/ADIOSMPI.h" #include "adios2/ADIOSTypes.h" #include "adios2/toolkit/profiling/iochrono/IOChrono.h" diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h index 8c3889a14e..0309a84367 100644 --- a/source/adios2/toolkit/transportman/TransportMan.h +++ b/source/adios2/toolkit/transportman/TransportMan.h @@ -18,7 +18,6 @@ #include /// \endcond -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/toolkit/transport/Transport.h" namespace adios2 diff --git a/source/utils/adios_reorganize/Reorganize.h b/source/utils/adios_reorganize/Reorganize.h index db1e3f159b..34274c2928 100644 --- a/source/utils/adios_reorganize/Reorganize.h +++ b/source/utils/adios_reorganize/Reorganize.h @@ -12,7 +12,6 @@ #define UTILS_REORGANIZE_REORGANIZE_H_ #include "adios2.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/core/IO.h" // DataMap #include "utils/Utils.h" diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index 4aabfc3eb9..fe8eea57a2 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -7,7 +7,6 @@ #include "adios2/ADIOSConfig.h" #include "adios2/ADIOSMPI.h" -#include "adios2/ADIOSMPICommOnly.h" #include "adios2/ADIOSMacros.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" From a656cbcf93250f693ec3da08d6d69ca5f5fc94a5 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 14:43:23 -0400 Subject: [PATCH 2/9] mpidummy: fix some bugs when getting sizeof MPI_Datatype --- source/adios2/helper/mpidummy.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp index de67549394..9c041c9aa8 100644 --- a/source/adios2/helper/mpidummy.cpp +++ b/source/adios2/helper/mpidummy.cpp @@ -140,19 +140,19 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, switch (recvtype) { case MPI_CHAR: - nrecv = sizeof(char); + n = sizeof(char); break; case MPI_INT: - nrecv = sizeof(int); + n = sizeof(int); break; case MPI_UNSIGNED: - nrecv = sizeof(unsigned int); + n = sizeof(unsigned int); break; case MPI_UNSIGNED_LONG: - nrecv = sizeof(unsigned long); + n = sizeof(unsigned long); break; case MPI_UNSIGNED_LONG_LONG: - nrecv = sizeof(unsigned long long); + n = sizeof(unsigned long long); break; default: return MPI_ERR_TYPE; @@ -230,7 +230,7 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, switch (recvtype) { case MPI_INT: - nrecv = sizeof(int); + n = sizeof(int); break; default: return MPI_ERR_TYPE; From 2f7852365c224d203367fe5d54231ba054b095ab Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 15:06:01 -0400 Subject: [PATCH 3/9] mpidummy: add MPI_Type_size() and use it internally, instead of repeated switch statements. I'm intentionally using an if-else chain rather than switch in the implementation, since I want this to work if MPI_Datatype isn't an integer type in the future. --- source/adios2/helper/mpidummy.cpp | 97 +++++++++++++++---------------- source/adios2/helper/mpidummy.h | 2 + 2 files changed, 48 insertions(+), 51 deletions(-) diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp index 9c041c9aa8..71d9ab0df4 100644 --- a/source/adios2/helper/mpidummy.cpp +++ b/source/adios2/helper/mpidummy.cpp @@ -105,57 +105,28 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, MPI_Comm comm) { int ier = MPI_SUCCESS; - size_t n = 0, nsent = 0, nrecv = 0; + int n; + size_t nsent = 0, nrecv = 0; if (!sendbuf && !recvbuf) { return ier; } if (comm == MPI_COMM_NULL || root) { - ier = MPI_ERR_COMM; + return MPI_ERR_COMM; } - switch (sendtype) + ier = MPI_Type_size(sendtype, &n); + if (ier != MPI_SUCCESS) { - case MPI_CHAR: - n = sizeof(char); - break; - case MPI_INT: - n = sizeof(int); - break; - case MPI_UNSIGNED: - n = sizeof(unsigned int); - break; - case MPI_UNSIGNED_LONG: - n = sizeof(unsigned long); - break; - case MPI_UNSIGNED_LONG_LONG: - n = sizeof(unsigned long long); - break; - default: - return MPI_ERR_TYPE; + return ier; } nsent = n * sendcnt; - switch (recvtype) + ier = MPI_Type_size(recvtype, &n); + if (ier != MPI_SUCCESS) { - case MPI_CHAR: - n = sizeof(char); - break; - case MPI_INT: - n = sizeof(int); - break; - case MPI_UNSIGNED: - n = sizeof(unsigned int); - break; - case MPI_UNSIGNED_LONG: - n = sizeof(unsigned long); - break; - case MPI_UNSIGNED_LONG_LONG: - n = sizeof(unsigned long long); - break; - default: - return MPI_ERR_TYPE; + return ier; } nrecv = n * recvcnt; @@ -206,7 +177,8 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, MPI_Comm comm) { int ier = MPI_SUCCESS; - size_t n = 0, nsent = 0, nrecv = 0; + int n; + size_t nsent = 0, nrecv = 0; if (!sendbuf || !recvbuf) { ier = MPI_ERR_BUFFER; @@ -217,23 +189,17 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, ier = MPI_ERR_COMM; } - switch (sendtype) + ier = MPI_Type_size(sendtype, &n); + if (ier != MPI_SUCCESS) { - case MPI_INT: - n = sizeof(int); - break; - default: - return MPI_ERR_TYPE; + return ier; } nsent = n * sendcnt; - switch (recvtype) + ier = MPI_Type_size(recvtype, &n); + if (ier != MPI_SUCCESS) { - case MPI_INT: - n = sizeof(int); - break; - default: - return MPI_ERR_TYPE; + return ier; } nrecv = n * recvcnt; @@ -467,6 +433,35 @@ int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, 0, comm); } +int MPI_Type_size(MPI_Datatype datatype, int *size) +{ + if (datatype == MPI_CHAR) + { + *size = sizeof(char); + } + else if (datatype == MPI_INT) + { + *size = sizeof(int); + } + else if (datatype == MPI_UNSIGNED) + { + *size = sizeof(unsigned int); + } + else if (datatype == MPI_UNSIGNED_LONG) + { + *size = sizeof(unsigned long); + } + else if (datatype == MPI_UNSIGNED_LONG_LONG) + { + *size = sizeof(unsigned long long); + } + else + { + return MPI_ERR_TYPE; + } + return MPI_SUCCESS; +} + } // end namespace mpi } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h index 693bb23ad7..06822ee77f 100644 --- a/source/adios2/helper/mpidummy.h +++ b/source/adios2/helper/mpidummy.h @@ -126,6 +126,8 @@ int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int MPI_Wait(MPI_Request *request, MPI_Status *status); +int MPI_Type_size(MPI_Datatype datatype, int *size); + int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh); int MPI_File_close(MPI_File *fh); From 5fe9b530d9fce987940d2f60f960dbdf4158451b Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 15:14:32 -0400 Subject: [PATCH 4/9] mpidummy: fix some returned error codes --- source/adios2/helper/mpidummy.cpp | 33 ++++++++++++++++++++++--------- source/adios2/helper/mpidummy.h | 1 + 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp index 71d9ab0df4..a1f0b17c5b 100644 --- a/source/adios2/helper/mpidummy.cpp +++ b/source/adios2/helper/mpidummy.cpp @@ -107,11 +107,19 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, int ier = MPI_SUCCESS; int n; size_t nsent = 0, nrecv = 0; - if (!sendbuf && !recvbuf) + if (sendcnt > 0 && !sendbuf) { - return ier; + return MPI_ERR_BUFFER; + } + if (recvcnt > 0 && !recvbuf) + { + return MPI_ERR_BUFFER; } - if (comm == MPI_COMM_NULL || root) + if (root != 0) + { + return MPI_ERR_ROOT; + } + if (comm == MPI_COMM_NULL) { return MPI_ERR_COMM; } @@ -155,7 +163,7 @@ int MPI_Gatherv(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, int ier = MPI_SUCCESS; if (*recvcnts != sendcnt) { - ier = MPI_ERR_BUFFER; + ier = MPI_ERR_COUNT; return ier; } @@ -179,14 +187,21 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, int ier = MPI_SUCCESS; int n; size_t nsent = 0, nrecv = 0; - if (!sendbuf || !recvbuf) + if (sendcnt > 0 && !sendbuf) { - ier = MPI_ERR_BUFFER; + return MPI_ERR_BUFFER; } - - if (comm == MPI_COMM_NULL || root) + if (recvcnt > 0 && !recvbuf) + { + return MPI_ERR_BUFFER; + } + if (root != 0) { - ier = MPI_ERR_COMM; + return MPI_ERR_ROOT; + } + if (comm == MPI_COMM_NULL) + { + return MPI_ERR_COMM; } ier = MPI_Type_size(sendtype, &n); diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h index 06822ee77f..130dbe965c 100644 --- a/source/adios2/helper/mpidummy.h +++ b/source/adios2/helper/mpidummy.h @@ -36,6 +36,7 @@ using MPI_Op = int; #define MPI_ERR_TYPE 3 /* Invalid datatype argument */ #define MPI_ERR_TAG 4 /* Invalid tag argument */ #define MPI_ERR_COMM 5 /* Invalid communicator */ +#define MPI_ERR_ROOT 6 /* Invalid root process */ #define MPI_ERR_INTERN 17 /* Invalid memory */ #define MPI_MAX_ERROR_STRING 512 #define MPI_MODE_RDONLY 1 From ebf6860b82b0ca41837edc88a3be49f674c6b2f3 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 15:21:25 -0400 Subject: [PATCH 5/9] mpidummy: get rid of MPI_Error_string It's never used, actually incorrectly implemented (it's supposed to just return string based on the passed-in error code, not maintain "last error" state), and as seen by a recent bug fix, still has the potential for buggy behavior. --- source/adios2/helper/mpidummy.cpp | 30 ------------------------------ source/adios2/helper/mpidummy.h | 1 - 2 files changed, 31 deletions(-) diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp index a1f0b17c5b..e1150e871f 100644 --- a/source/adios2/helper/mpidummy.cpp +++ b/source/adios2/helper/mpidummy.cpp @@ -34,17 +34,13 @@ namespace helper namespace mpi { -static char mpierrmsg[MPI_MAX_ERROR_STRING]; - int MPI_Init(int * /*argc*/, char *** /*argv*/) { - mpierrmsg[0] = '\0'; return MPI_SUCCESS; } int MPI_Finalize() { - mpierrmsg[0] = '\0'; return MPI_SUCCESS; } @@ -147,11 +143,6 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, { std::memcpy(recvbuf, sendbuf, nsent); } - else - { - std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, - "could not gather data\n"); - } return ier; } @@ -227,11 +218,6 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, { std::memcpy(recvbuf, sendbuf, nsent); } - else - { - std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, - "could not scatter data\n"); - } return ier; } @@ -306,8 +292,6 @@ int MPI_File_open(MPI_Comm /*comm*/, const char *filename, int amode, *fh = std::fopen(filename, mode.c_str()); if (!*fh) { - std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, "File not found: %s", - filename); return -1; } return MPI_SUCCESS; @@ -335,11 +319,6 @@ int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, bytes_read = std::fread(buf, 1, bytes_to_read, fh); if (bytes_read != bytes_to_read) { - std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, - "could not read %llu bytes. read only: %llu" - "\n", - (unsigned long long)bytes_to_read, - (unsigned long long)bytes_read); return -2; } *status = bytes_read; @@ -359,15 +338,6 @@ int MPI_Get_count(const MPI_Status *status, MPI_Datatype, int *count) return MPI_SUCCESS; } -int MPI_Error_string(int /*errorcode*/, char *string, int *resultlen) -{ - // std::sprintf(string, "Dummy lib does not know error strings. - // Code=%d\n",errorcode); - std::strcpy(string, mpierrmsg); - *resultlen = static_cast(std::strlen(string)); - return MPI_SUCCESS; -} - double MPI_Wtime() { std::chrono::duration now = diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h index 130dbe965c..46eb08db10 100644 --- a/source/adios2/helper/mpidummy.h +++ b/source/adios2/helper/mpidummy.h @@ -138,7 +138,6 @@ int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence); int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count); -int MPI_Error_string(int errorcode, char *string, int *resultlen); int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *comm_out); int MPI_Get_processor_name(char *name, int *resultlen); From d5365eb29489128ffa68b65b76f5d96b5d9c15ba Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 29 May 2019 21:37:03 -0400 Subject: [PATCH 6/9] mpiwrap: provide SMPI_* wrappers around MPI that handle serial case In the serial case, these wrappers will always call through to the mpidummy MPI functons. In the case where adios2 is compiled with MPI support, the SMPI wrappers will call through to the actual MPI implementation, if MPI has been initialized (which is currently always the case, otherwise one would get errors from the MPI library). The new behavior is that the wrappers catch the case where MPI has not been initialized and fall back to mpidummy in this case. --- source/adios2/ADIOSMPI.h | 7 +- source/adios2/CMakeLists.txt | 4 +- source/adios2/helper/adiosMPIFunctions.tcc | 22 ++-- source/adios2/helper/mpidummy.cpp | 70 ++++++----- source/adios2/helper/mpidummy.h | 25 +++- source/adios2/helper/mpiwrap.cpp | 135 +++++++++++++++++++++ source/adios2/helper/mpiwrap.h | 42 +++++++ 7 files changed, 252 insertions(+), 53 deletions(-) create mode 100644 source/adios2/helper/mpiwrap.cpp create mode 100644 source/adios2/helper/mpiwrap.h diff --git a/source/adios2/ADIOSMPI.h b/source/adios2/ADIOSMPI.h index fc4bd235dc..8190eb3ed1 100644 --- a/source/adios2/ADIOSMPI.h +++ b/source/adios2/ADIOSMPI.h @@ -8,12 +8,7 @@ #include "adios2/ADIOSConfig.h" -#ifdef ADIOS2_HAVE_MPI -#include -#else -#include "adios2/helper/mpidummy.h" -using namespace adios2::helper::mpi; -#endif +#include "adios2/helper/mpiwrap.h" #include //UXXX_MAX #include //SIZE_MAX diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index d282636a27..2c82e4f947 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -186,10 +186,10 @@ if(ADIOS2_HAVE_MPI) engine/insitumpi/InSituMPIFunctions.cpp engine/insitumpi/InSituMPISchedules.cpp ) target_link_libraries(adios2 PUBLIC MPI::MPI_C) -else() - target_sources(adios2 PRIVATE helper/mpidummy.cpp) endif() +target_sources(adios2 PRIVATE helper/mpidummy.cpp helper/mpiwrap.cpp) + if(ADIOS2_HAVE_HDF5) if(HDF5_C_INCLUDE_DIRS) target_include_directories(adios2 PRIVATE ${HDF5_C_INCLUDE_DIRS}) diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc index fd793d7f62..7e3a56a924 100644 --- a/source/adios2/helper/adiosMPIFunctions.tcc +++ b/source/adios2/helper/adiosMPIFunctions.tcc @@ -40,7 +40,7 @@ size_t BroadcastValue(const size_t &input, MPI_Comm mpiComm, output = input; } - MPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm); + SMPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm); return output; } @@ -64,8 +64,8 @@ std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm, output.resize(length); } - MPI_Bcast(const_cast(output.data()), static_cast(length), - MPI_CHAR, rankSource, mpiComm); + SMPI_Bcast(const_cast(output.data()), static_cast(length), + MPI_CHAR, rankSource, mpiComm); return output; } @@ -77,8 +77,8 @@ unsigned int ReduceValues(const unsigned int source, MPI_Comm mpiComm, { unsigned int sourceLocal = source; unsigned int reduceValue = 0; - MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation, - rankDestination, mpiComm); + SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation, + rankDestination, mpiComm); return reduceValue; } @@ -88,8 +88,8 @@ unsigned long int ReduceValues(const unsigned long int source, MPI_Comm mpiComm, { unsigned long int sourceLocal = source; unsigned long int reduceValue = 0; - MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation, - rankDestination, mpiComm); + SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation, + rankDestination, mpiComm); return reduceValue; } @@ -100,8 +100,8 @@ unsigned long long int ReduceValues(const unsigned long long int source, { unsigned long long int sourceLocal = source; unsigned long long int reduceValue = 0; - MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG, operation, - rankDestination, mpiComm); + SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG, + operation, rankDestination, mpiComm); return reduceValue; } @@ -133,8 +133,8 @@ void BroadcastVector(std::vector &vector, MPI_Comm mpiComm, char *buffer = vector.data(); while (inputSize > 0) { - MPI_Bcast(buffer, static_cast(blockSize), MPI_CHAR, rankSource, - mpiComm); + SMPI_Bcast(buffer, static_cast(blockSize), MPI_CHAR, rankSource, + mpiComm); buffer += blockSize; inputSize -= blockSize; blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp index e1150e871f..32eaf48cd6 100644 --- a/source/adios2/helper/mpidummy.cpp +++ b/source/adios2/helper/mpidummy.cpp @@ -31,18 +31,12 @@ namespace adios2 { namespace helper { -namespace mpi +namespace mpidummy { -int MPI_Init(int * /*argc*/, char *** /*argv*/) -{ - return MPI_SUCCESS; -} +int MPI_Init(int * /*argc*/, char *** /*argv*/) { return MPI_SUCCESS; } -int MPI_Finalize() -{ - return MPI_SUCCESS; -} +int MPI_Finalize() { return MPI_SUCCESS; } int MPI_Initialized(int *flag) { @@ -94,7 +88,9 @@ int MPI_Comm_free(MPI_Comm *comm) return MPI_SUCCESS; } +#ifndef ADIOS2_HAVE_MPI MPI_Comm MPI_Comm_f2c(MPI_Fint comm) { return comm; } +#endif int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, void *recvbuf, int recvcnt, MPI_Datatype recvtype, int root, @@ -120,14 +116,14 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, return MPI_ERR_COMM; } - ier = MPI_Type_size(sendtype, &n); + ier = mpidummy::MPI_Type_size(sendtype, &n); if (ier != MPI_SUCCESS) { return ier; } nsent = n * sendcnt; - ier = MPI_Type_size(recvtype, &n); + ier = mpidummy::MPI_Type_size(recvtype, &n); if (ier != MPI_SUCCESS) { return ier; @@ -158,8 +154,8 @@ int MPI_Gatherv(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, return ier; } - ier = MPI_Gather(sendbuf, sendcnt, sendtype, recvbuf, *recvcnts, recvtype, - root, comm); + ier = mpidummy::MPI_Gather(sendbuf, sendcnt, sendtype, recvbuf, *recvcnts, + recvtype, root, comm); return ier; } @@ -167,8 +163,8 @@ int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { - return MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, 0, comm); + return mpidummy::MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, + recvcount, recvtype, 0, comm); } int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, @@ -195,14 +191,14 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype, return MPI_ERR_COMM; } - ier = MPI_Type_size(sendtype, &n); + ier = mpidummy::MPI_Type_size(sendtype, &n); if (ier != MPI_SUCCESS) { return ier; } nsent = n * sendcnt; - ier = MPI_Type_size(recvtype, &n); + ier = mpidummy::MPI_Type_size(recvtype, &n); if (ier != MPI_SUCCESS) { return ier; @@ -234,8 +230,8 @@ int MPI_Scatterv(const void *sendbuf, const int *sendcnts, const int *displs, if (ier == MPI_SUCCESS) { - ier = MPI_Scatter(sendbuf, *sendcnts, sendtype, recvbuf, recvcnt, - recvtype, root, comm); + ier = mpidummy::MPI_Scatter(sendbuf, *sendcnts, sendtype, recvbuf, + recvcnt, recvtype, root, comm); } return ier; @@ -271,6 +267,8 @@ int MPI_Isend(const void * /*recvbuffer*/, int /*count*/, MPI_Datatype /*type*/, int MPI_Wait(MPI_Request * /*request*/, MPI_Status * /*status*/) { return 0; } +#ifndef ADIOS2_HAVE_MPI + int MPI_File_open(MPI_Comm /*comm*/, const char *filename, int amode, MPI_Info /*info*/, MPI_File *fh) { @@ -338,6 +336,8 @@ int MPI_Get_count(const MPI_Status *status, MPI_Datatype, int *count) return MPI_SUCCESS; } +#endif + double MPI_Wtime() { std::chrono::duration now = @@ -355,25 +355,26 @@ int MPI_Get_processor_name(char *name, int *resultlen) int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) { - switch (datatype) + if (datatype == MPI_CHAR) { - case MPI_CHAR: if (op == MPI_SUM) { char *recvBuffer = reinterpret_cast(recvbuf); const char *sendBuffer = reinterpret_cast(sendbuf); *recvBuffer = std::accumulate(sendBuffer, sendBuffer + count, 0); } - break; - case MPI_INT: + } + else if (datatype == MPI_INT) + { if (op == MPI_SUM) { int *recvBuffer = reinterpret_cast(recvbuf); const int *sendBuffer = reinterpret_cast(sendbuf); *recvBuffer = std::accumulate(sendBuffer, sendBuffer + count, 0); } - break; - case MPI_UNSIGNED: + } + else if (datatype == MPI_UNSIGNED) + { if (op == MPI_SUM) { unsigned int *recvBuffer = @@ -382,8 +383,9 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, reinterpret_cast(sendbuf); *recvBuffer = std::accumulate(sendBuffer, sendBuffer + count, 0); } - break; - case MPI_UNSIGNED_LONG: + } + else if (datatype == MPI_UNSIGNED_LONG) + { if (op == MPI_SUM) { unsigned long int *recvBuffer = @@ -392,8 +394,9 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, reinterpret_cast(sendbuf); *recvBuffer = std::accumulate(sendBuffer, sendBuffer + count, 0); } - break; - case MPI_UNSIGNED_LONG_LONG: + } + else if (datatype == MPI_UNSIGNED_LONG_LONG) + { if (op == MPI_SUM) { unsigned long long int *recvBuffer = @@ -404,8 +407,9 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, std::accumulate(sendBuffer, sendBuffer + count, static_cast(0)); } - break; - default: + } + else + { return MPI_ERR_TYPE; } @@ -415,7 +419,7 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) { - return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, 0, comm); + return mpidummy::MPI_Reduce(sendbuf, recvbuf, count, datatype, op, 0, comm); } int MPI_Type_size(MPI_Datatype datatype, int *size) @@ -447,6 +451,6 @@ int MPI_Type_size(MPI_Datatype datatype, int *size) return MPI_SUCCESS; } -} // end namespace mpi +} // end namespace mpidummy } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h index 46eb08db10..44daa3e60b 100644 --- a/source/adios2/helper/mpidummy.h +++ b/source/adios2/helper/mpidummy.h @@ -13,11 +13,20 @@ #include #include +#include "adios2/ADIOSConfig.h" + +// If MPI is available, use the types and constants from that implementation, +// rather than our fake ones, though we stll provide the fake implementations +// (in a separate namespace) +#ifdef ADIOS2_HAVE_MPI +#include +#else + namespace adios2 { namespace helper { -namespace mpi +namespace mpidummy { using MPI_Comm = int; @@ -83,6 +92,18 @@ using MPI_Op = int; #define MPI_MAX 1 #define MPI_MAX_PROCESSOR_NAME 32 +} +} +} + +#endif + +namespace adios2 +{ +namespace helper +{ +namespace mpidummy +{ int MPI_Init(int *argc, char ***argv); int MPI_Finalize(); @@ -97,7 +118,9 @@ int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); int MPI_Comm_rank(MPI_Comm comm, int *rank); int MPI_Comm_size(MPI_Comm comm, int *size); int MPI_Comm_free(MPI_Comm *comm); +#ifndef ADIOS2_HAVE_MPI MPI_Comm MPI_Comm_f2c(MPI_Fint comm); +#endif int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, diff --git a/source/adios2/helper/mpiwrap.cpp b/source/adios2/helper/mpiwrap.cpp new file mode 100644 index 0000000000..ea3040ad38 --- /dev/null +++ b/source/adios2/helper/mpiwrap.cpp @@ -0,0 +1,135 @@ + +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + */ + +#include "mpiwrap.h" + +static inline bool SMPI_Available() +{ +#ifdef ADIOS2_HAVE_MPI + int flag; + ::MPI_Initialized(&flag); + if (!flag) + { + return false; + } + ::MPI_Finalized(&flag); + return !flag; +#else + // if compiled without MPI, always fall back to mpidummy + return false; +#endif +} + +#define MAKE_MPI_WRAPPER1(NAME, TARG1, ARG1) \ + int S##NAME(TARG1 ARG1) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME(ARG1); \ + } \ + } + +#define MAKE_MPI_WRAPPER2(NAME, TARG1, ARG1, TARG2, ARG2) \ + int S##NAME(TARG1 ARG1, TARG2 ARG2) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1, ARG2); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME(ARG1, ARG2); \ + } \ + } + +#define MAKE_MPI_WRAPPER5(NAME, TARG1, ARG1, TARG2, ARG2, TARG3, ARG3, TARG4, \ + ARG4, TARG5, ARG5) \ + int S##NAME(TARG1 ARG1, TARG2 ARG2, TARG3 ARG3, TARG4 ARG4, TARG5 ARG5) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1, ARG2, ARG3, ARG4, ARG5); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME(ARG1, ARG2, ARG3, ARG4, \ + ARG5); \ + } \ + } + +#define MAKE_MPI_WRAPPER7(NAME, TARG1, ARG1, TARG2, ARG2, TARG3, ARG3, TARG4, \ + ARG4, TARG5, ARG5, TARG6, ARG6, TARG7, ARG7) \ + int S##NAME(TARG1 ARG1, TARG2 ARG2, TARG3 ARG3, TARG4 ARG4, TARG5 ARG5, \ + TARG6 ARG6, TARG7 ARG7) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME(ARG1, ARG2, ARG3, ARG4, \ + ARG5, ARG6, ARG7); \ + } \ + } + +#define MAKE_MPI_WRAPPER8(NAME, TARG1, ARG1, TARG2, ARG2, TARG3, ARG3, TARG4, \ + ARG4, TARG5, ARG5, TARG6, ARG6, TARG7, ARG7, TARG8, \ + ARG8) \ + int S##NAME(TARG1 ARG1, TARG2 ARG2, TARG3 ARG3, TARG4 ARG4, TARG5 ARG5, \ + TARG6 ARG6, TARG7 ARG7, TARG8 ARG8) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME(ARG1, ARG2, ARG3, ARG4, \ + ARG5, ARG6, ARG7, ARG8); \ + } \ + } + +#define MAKE_MPI_WRAPPER9(NAME, TARG1, ARG1, TARG2, ARG2, TARG3, ARG3, TARG4, \ + ARG4, TARG5, ARG5, TARG6, ARG6, TARG7, ARG7, TARG8, \ + ARG8, TARG9, ARG9) \ + int S##NAME(TARG1 ARG1, TARG2 ARG2, TARG3 ARG3, TARG4 ARG4, TARG5 ARG5, \ + TARG6 ARG6, TARG7 ARG7, TARG8 ARG8, TARG9 ARG9) \ + { \ + if (SMPI_Available()) \ + { \ + return ::NAME(ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, \ + ARG9); \ + } \ + else \ + { \ + return adios2::helper::mpidummy::NAME( \ + ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9); \ + } \ + } + +MAKE_MPI_WRAPPER2(MPI_Comm_dup, MPI_Comm, comm, MPI_Comm *, newcomm) +MAKE_MPI_WRAPPER1(MPI_Comm_free, MPI_Comm *, newcomm) +MAKE_MPI_WRAPPER2(MPI_Comm_rank, MPI_Comm, comm, int *, rank) +MAKE_MPI_WRAPPER2(MPI_Comm_size, MPI_Comm, comm, int *, size) +MAKE_MPI_WRAPPER1(MPI_Barrier, MPI_Comm, comm) +MAKE_MPI_WRAPPER5(MPI_Bcast, void *, buffer, int, count, MPI_Datatype, datatype, + int, root, MPI_Comm, comm) +MAKE_MPI_WRAPPER8(MPI_Gather, const void *, sendbuf, int, sendcount, + MPI_Datatype, sendtype, void *, recvbuf, int, recvcount, + MPI_Datatype, recvtype, int, root, MPI_Comm, comm) +MAKE_MPI_WRAPPER9(MPI_Gatherv, const void *, sendbuf, int, sendcount, + MPI_Datatype, sendtype, void *, recvbuf, const int *, + recvcounts, const int *, displs, MPI_Datatype, recvtype, int, + root, MPI_Comm, comm) +MAKE_MPI_WRAPPER7(MPI_Reduce, const void *, sendbuf, void *, recvbuf, int, + count, MPI_Datatype, datatype, MPI_Op, op, int, root, + MPI_Comm, comm) diff --git a/source/adios2/helper/mpiwrap.h b/source/adios2/helper/mpiwrap.h new file mode 100644 index 0000000000..d1afa0fc5d --- /dev/null +++ b/source/adios2/helper/mpiwrap.h @@ -0,0 +1,42 @@ + +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * + */ + +#ifndef ADIOS2_HELPER_MPIWRAP_H_ +#define ADIOS2_HELPER_MPIWRAP_H_ + +// Get either the real MPI or mpidummy into the global namespace +#ifdef ADIOS2_HAVE_MPI +#include +#else +#include "mpidummy.h" +using namespace adios2::helper::mpidummy; +#endif + +// SMPI_* work just as the original MPI_* functions, but will also +// work if MPI_Init has not been called, as long asssuming the communicator is a +// single-proc communicator, by falling back to mpidummy in that case + +extern "C" { +int SMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); +int SMPI_Comm_free(MPI_Comm *newcomm); +int SMPI_Comm_rank(MPI_Comm comm, int *rank); +int SMPI_Comm_size(MPI_Comm comm, int *size); +int SMPI_Barrier(MPI_Comm comm); +int SMPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, + MPI_Comm comm); +int SMPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, + MPI_Comm comm); +int SMPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, int root, MPI_Comm comm); +int SMPI_Reduce(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm); +} + +#endif From 3991b322602b049d2f5d6056ed218e6b9739f222 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 31 May 2019 11:31:07 -0400 Subject: [PATCH 7/9] mpiwrap: use SMPI_ to make BP3 read/write and HDF5 read work --- source/adios2/core/Engine.cpp | 2 +- source/adios2/core/IO.cpp | 2 +- source/adios2/helper/adiosMPIFunctions.inl | 8 ++-- source/adios2/helper/adiosMPIFunctions.tcc | 37 ++++++++++--------- source/adios2/toolkit/format/bp3/BP3Base.cpp | 4 +- .../toolkit/format/bp3/BP3Serializer.cpp | 6 +-- source/adios2/toolkit/format/bp4/BP4Base.cpp | 4 +- .../toolkit/format/bp4/BP4Serializer.cpp | 10 ++--- .../toolkit/interop/hdf5/HDF5Common.cpp | 9 +++-- source/adios2/toolkit/transport/Transport.cpp | 4 +- .../toolkit/transportman/TransportMan.cpp | 4 +- 11 files changed, 47 insertions(+), 43 deletions(-) diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 5877b8fa4e..b6463486a7 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -69,7 +69,7 @@ void Engine::Close(const int transportIndex) if (transportIndex == -1) { - helper::CheckMPIReturn(MPI_Comm_free(&m_MPIComm), + helper::CheckMPIReturn(SMPI_Comm_free(&m_MPIComm), "freeing comm in Engine " + m_Name + ", in call to Close"); m_IsClosed = true; diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 238c8c93dd..2bd4f1ef0b 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -438,7 +438,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, } MPI_Comm mpiComm; - MPI_Comm_dup(mpiComm_orig, &mpiComm); + SMPI_Comm_dup(mpiComm_orig, &mpiComm); std::shared_ptr engine; const bool isDefaultEngine = m_EngineType.empty() ? true : false; std::string engineTypeLC = m_EngineType; diff --git a/source/adios2/helper/adiosMPIFunctions.inl b/source/adios2/helper/adiosMPIFunctions.inl index 468d1d7df0..ed7322595f 100644 --- a/source/adios2/helper/adiosMPIFunctions.inl +++ b/source/adios2/helper/adiosMPIFunctions.inl @@ -27,8 +27,8 @@ std::vector GatherValues(const T source, MPI_Comm mpiComm, const int rankDestination) { int rank, size; - MPI_Comm_rank(mpiComm, &rank); - MPI_Comm_size(mpiComm, &size); + SMPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_size(mpiComm, &size); std::vector output; @@ -47,7 +47,7 @@ template std::vector AllGatherValues(const T source, MPI_Comm mpiComm) { int size; - MPI_Comm_size(mpiComm, &size); + SMPI_Comm_size(mpiComm, &size); std::vector output(size); T sourceCopy = source; // so we can have an address for rvalues @@ -67,7 +67,7 @@ void GathervVectors(const std::vector &in, std::vector &out, size_t gatheredSize = 0; int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); if (rank == rankDestination) // pre-allocate vector { diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc index 7e3a56a924..fbb5e13e43 100644 --- a/source/adios2/helper/adiosMPIFunctions.tcc +++ b/source/adios2/helper/adiosMPIFunctions.tcc @@ -32,7 +32,7 @@ size_t BroadcastValue(const size_t &input, MPI_Comm mpiComm, const int rankSource) { int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); size_t output = 0; if (rank == rankSource) @@ -50,7 +50,7 @@ std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm, const int rankSource) { int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); const size_t inputSize = input.size(); const size_t length = BroadcastValue(inputSize, mpiComm, rankSource); std::string output; @@ -111,7 +111,7 @@ void BroadcastVector(std::vector &vector, MPI_Comm mpiComm, const int rankSource) { int size; - MPI_Comm_size(mpiComm, &size); + SMPI_Comm_size(mpiComm, &size); if (size == 1) { @@ -121,7 +121,7 @@ void BroadcastVector(std::vector &vector, MPI_Comm mpiComm, // First Broadcast the size, then the contents size_t inputSize = BroadcastValue(vector.size(), mpiComm, rankSource); int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); if (rank != rankSource) { @@ -149,8 +149,8 @@ void GatherArrays(const char *source, const size_t sourceCount, { int countsInt = static_cast(sourceCount); int result = - MPI_Gather(const_cast(source), countsInt, MPI_CHAR, destination, - countsInt, MPI_CHAR, rankDestination, mpiComm); + SMPI_Gather(const_cast(source), countsInt, MPI_CHAR, + destination, countsInt, MPI_CHAR, rankDestination, mpiComm); if (result != MPI_SUCCESS) { @@ -165,9 +165,9 @@ void GatherArrays(const size_t *source, const size_t sourceCount, const int rankDestination) { int countsInt = static_cast(sourceCount); - int result = MPI_Gather(const_cast(source), countsInt, - ADIOS2_MPI_SIZE_T, destination, countsInt, - ADIOS2_MPI_SIZE_T, rankDestination, mpiComm); + int result = SMPI_Gather(const_cast(source), countsInt, + ADIOS2_MPI_SIZE_T, destination, countsInt, + ADIOS2_MPI_SIZE_T, rankDestination, mpiComm); if (result != MPI_SUCCESS) { @@ -202,7 +202,7 @@ void GathervArrays(const char *source, const size_t sourceCount, { int result = 0; int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); std::vector countsInt, displacementsInt; @@ -213,9 +213,10 @@ void GathervArrays(const char *source, const size_t sourceCount, } int sourceCountInt = static_cast(sourceCount); - result = MPI_Gatherv(const_cast(source), sourceCountInt, MPI_CHAR, - destination, countsInt.data(), displacementsInt.data(), - MPI_CHAR, rankDestination, mpiComm); + result = + SMPI_Gatherv(const_cast(source), sourceCountInt, MPI_CHAR, + destination, countsInt.data(), displacementsInt.data(), + MPI_CHAR, rankDestination, mpiComm); if (result != MPI_SUCCESS) { @@ -232,7 +233,7 @@ void GathervArrays(const size_t *source, const size_t sourceCount, { int result = 0; int rank; - MPI_Comm_rank(mpiComm, &rank); + SMPI_Comm_rank(mpiComm, &rank); std::vector countsInt = NewVectorTypeFromArray(counts, countsSize); @@ -242,10 +243,10 @@ void GathervArrays(const size_t *source, const size_t sourceCount, int sourceCountInt = static_cast(sourceCount); - result = MPI_Gatherv(const_cast(source), sourceCountInt, - ADIOS2_MPI_SIZE_T, destination, countsInt.data(), - displacementsInt.data(), ADIOS2_MPI_SIZE_T, - rankDestination, mpiComm); + result = SMPI_Gatherv(const_cast(source), sourceCountInt, + ADIOS2_MPI_SIZE_T, destination, countsInt.data(), + displacementsInt.data(), ADIOS2_MPI_SIZE_T, + rankDestination, mpiComm); if (result != MPI_SUCCESS) { diff --git a/source/adios2/toolkit/format/bp3/BP3Base.cpp b/source/adios2/toolkit/format/bp3/BP3Base.cpp index 3f484cb486..cf9df6fabc 100644 --- a/source/adios2/toolkit/format/bp3/BP3Base.cpp +++ b/source/adios2/toolkit/format/bp3/BP3Base.cpp @@ -47,8 +47,8 @@ const std::map BP3Base::m_TransformTypesToNames = { BP3Base::BP3Base(MPI_Comm mpiComm, const bool debugMode) : m_MPIComm(mpiComm), m_DebugMode(debugMode) { - MPI_Comm_rank(m_MPIComm, &m_RankMPI); - MPI_Comm_size(m_MPIComm, &m_SizeMPI); + SMPI_Comm_rank(m_MPIComm, &m_RankMPI); + SMPI_Comm_size(m_MPIComm, &m_SizeMPI); m_Profiler.IsActive = true; // default } diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp index cc11ad4d30..bc62dba7e4 100644 --- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp +++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp @@ -282,7 +282,7 @@ void BP3Serializer::AggregateCollectiveMetadata(MPI_Comm comm, AggregateCollectiveMetadataIndices(comm, bufferSTL); int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); if (rank == 0) { PutMinifooter(static_cast(indicesPosition[0]), @@ -770,8 +770,8 @@ BP3Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm, { TAU_SCOPED_TIMER_FUNC(); int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); + SMPI_Comm_rank(comm, &rank); + SMPI_Comm_size(comm, &size); // pre-allocate with rank 0 data size_t pgCount = 0; //< tracks global PG count diff --git a/source/adios2/toolkit/format/bp4/BP4Base.cpp b/source/adios2/toolkit/format/bp4/BP4Base.cpp index a55c4642ae..1024536950 100644 --- a/source/adios2/toolkit/format/bp4/BP4Base.cpp +++ b/source/adios2/toolkit/format/bp4/BP4Base.cpp @@ -48,8 +48,8 @@ const std::map BP4Base::m_TransformTypesToNames = { BP4Base::BP4Base(MPI_Comm mpiComm, const bool debugMode) : m_MPIComm(mpiComm), m_DebugMode(debugMode) { - MPI_Comm_rank(m_MPIComm, &m_RankMPI); - MPI_Comm_size(m_MPIComm, &m_SizeMPI); + SMPI_Comm_rank(m_MPIComm, &m_RankMPI); + SMPI_Comm_size(m_MPIComm, &m_SizeMPI); m_Profiler.IsActive = true; // default } diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp index b14e0ac1a1..5ac80c3d4b 100644 --- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp +++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp @@ -335,7 +335,7 @@ void BP4Serializer::AggregateCollectiveMetadata(MPI_Comm comm, AggregateMergeIndex(m_MetadataSet.AttributesIndices, comm, bufferSTL, true); int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); if (rank == 0) { /* no more minifooter in the global metadata*/ @@ -821,7 +821,7 @@ void BP4Serializer::AggregateIndex(const SerialElementIndex &index, auto &buffer = bufferSTL.m_Buffer; auto &position = bufferSTL.m_Position; int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); size_t countPosition = position; const size_t totalCount = helper::ReduceValues(count, comm); @@ -872,7 +872,7 @@ void BP4Serializer::AggregateMergeIndex( std::vector().swap(gatheredSerialIndices); int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); if (rank == 0) { @@ -922,7 +922,7 @@ std::vector BP4Serializer::SerializeIndices( serializedIndices.reserve(serializedIndicesSize); int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); for (const auto &indexPair : indices) { @@ -1045,7 +1045,7 @@ BP4Serializer::DeserializeIndicesPerRankThreads( // BODY OF FUNCTION starts here const size_t serializedSize = serialized.size(); int rank; - MPI_Comm_rank(comm, &rank); + SMPI_Comm_rank(comm, &rank); if (rank != 0 || serializedSize < 8) { diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 47b29b6005..fbbddc7727 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -117,9 +117,12 @@ void HDF5Common::Init(const std::string &name, MPI_Comm comm, bool toWrite) m_PropertyListId = H5Pcreate(H5P_FILE_ACCESS); #ifdef ADIOS2_HAVE_MPI - H5Pset_fapl_mpio(m_PropertyListId, comm, MPI_INFO_NULL); - MPI_Comm_rank(comm, &m_CommRank); - MPI_Comm_size(comm, &m_CommSize); + SMPI_Comm_rank(comm, &m_CommRank); + SMPI_Comm_size(comm, &m_CommSize); + if (m_CommSize != 1) + { + H5Pset_fapl_mpio(m_PropertyListId, comm, MPI_INFO_NULL); + } #endif // std::string ts0 = "/AdiosStep0"; diff --git a/source/adios2/toolkit/transport/Transport.cpp b/source/adios2/toolkit/transport/Transport.cpp index 83e8e8ca84..745e3014f5 100644 --- a/source/adios2/toolkit/transport/Transport.cpp +++ b/source/adios2/toolkit/transport/Transport.cpp @@ -20,8 +20,8 @@ Transport::Transport(const std::string type, const std::string library, MPI_Comm mpiComm, const bool debugMode) : m_Type(type), m_Library(library), m_MPIComm(mpiComm), m_DebugMode(debugMode) { - MPI_Comm_rank(m_MPIComm, &m_RankMPI); - MPI_Comm_size(m_MPIComm, &m_SizeMPI); + SMPI_Comm_rank(m_MPIComm, &m_RankMPI); + SMPI_Comm_size(m_MPIComm, &m_SizeMPI); } void Transport::IWrite(const char *buffer, size_t size, Status &status, diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index b7b37c6d4d..796dca7951 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -58,13 +58,13 @@ void TransportMan::MkDirsBarrier(const std::vector &fileNames, else { int rank; - MPI_Comm_rank(m_MPIComm, &rank); + SMPI_Comm_rank(m_MPIComm, &rank); if (rank == 0) { lf_CreateDirectories(fileNames); } - helper::CheckMPIReturn(MPI_Barrier(m_MPIComm), + helper::CheckMPIReturn(SMPI_Barrier(m_MPIComm), "Barrier in TransportMan.MkDirsBarrier"); } } From c5928e713b5144c7e2b749f162389fb2499d1859 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 24 May 2019 15:17:23 -0400 Subject: [PATCH 8/9] mpiwrap: provide serial ADIOS constructors even when compiled with mpi and add a simple test --- bindings/CXX11/cxx11/ADIOS.cpp | 3 +-- bindings/CXX11/cxx11/ADIOS.h | 5 ++-- testing/adios2/interface/CMakeLists.txt | 4 +++ testing/adios2/interface/TestADIOSNoMpi.cpp | 29 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 testing/adios2/interface/TestADIOSNoMpi.cpp diff --git a/bindings/CXX11/cxx11/ADIOS.cpp b/bindings/CXX11/cxx11/ADIOS.cpp index 1d76144024..31a2b30c71 100644 --- a/bindings/CXX11/cxx11/ADIOS.cpp +++ b/bindings/CXX11/cxx11/ADIOS.cpp @@ -25,14 +25,13 @@ ADIOS::ADIOS(MPI_Comm comm, const bool debugMode) : ADIOS("", comm, debugMode) { } -#else +#endif ADIOS::ADIOS(const std::string &configFile, const bool debugMode) : m_ADIOS(std::make_shared(configFile, debugMode, "C++")) { } ADIOS::ADIOS(const bool debugMode) : ADIOS("", debugMode) {} -#endif ADIOS::operator bool() const noexcept { return m_ADIOS ? true : false; } diff --git a/bindings/CXX11/cxx11/ADIOS.h b/bindings/CXX11/cxx11/ADIOS.h index dff4a59937..f11328b741 100644 --- a/bindings/CXX11/cxx11/ADIOS.h +++ b/bindings/CXX11/cxx11/ADIOS.h @@ -64,9 +64,9 @@ class ADIOS * @exception std::invalid_argument in debugMode = true if user input is * incorrect */ - ADIOS(const std::string &configFile = "", MPI_Comm comm = MPI_COMM_SELF, + ADIOS(const std::string &configFile, MPI_Comm comm, const bool debugMode = true); -#else +#endif /** * Starting point for non-MPI serial apps. Creates an ADIOS object allowing @@ -87,7 +87,6 @@ class ADIOS * incorrect */ ADIOS(const bool debugMode = true); -#endif /** object inspection true: valid object, false: invalid object */ explicit operator bool() const noexcept; diff --git a/testing/adios2/interface/CMakeLists.txt b/testing/adios2/interface/CMakeLists.txt index b036eee798..e686e9761c 100644 --- a/testing/adios2/interface/CMakeLists.txt +++ b/testing/adios2/interface/CMakeLists.txt @@ -15,6 +15,9 @@ target_link_libraries(TestADIOSDefineVariable adios2 gtest) add_executable(TestADIOSDefineAttribute TestADIOSDefineAttribute.cpp) target_link_libraries(TestADIOSDefineAttribute adios2 gtest) +add_executable(TestADIOSNoMpi TestADIOSNoMpi.cpp) +target_link_libraries(TestADIOSNoMpi adios2 gtest) + if(ADIOS2_HAVE_MPI) target_link_libraries(TestADIOSInterface MPI::MPI_C) target_link_libraries(TestADIOSInterfaceWrite MPI::MPI_C) @@ -28,3 +31,4 @@ gtest_add_tests(TARGET TestADIOSInterface ${extra_test_args}) gtest_add_tests(TARGET TestADIOSInterfaceWrite ${extra_test_args}) gtest_add_tests(TARGET TestADIOSDefineVariable ${extra_test_args}) gtest_add_tests(TARGET TestADIOSDefineAttribute ${extra_test_args}) +gtest_add_tests(TARGET TestADIOSNoMpi) diff --git a/testing/adios2/interface/TestADIOSNoMpi.cpp b/testing/adios2/interface/TestADIOSNoMpi.cpp new file mode 100644 index 0000000000..8a05ec9c09 --- /dev/null +++ b/testing/adios2/interface/TestADIOSNoMpi.cpp @@ -0,0 +1,29 @@ + +#include + +#include + +/** + * Basic test to see that writing and reading a simple BP3 file works without + * initializing MPI + */ +TEST(ADIOSInterface, ADIOSNoMpi) +{ + adios2::ADIOS adios(adios2::DebugON); + { + adios2::IO io = adios.DeclareIO("TestIOWrite"); + adios2::Engine engine = io.Open("test.bp", adios2::Mode::Write); + engine.Close(); + } + { + adios2::IO io = adios.DeclareIO("TestIORead"); + adios2::Engine engine = io.Open("test.bp", adios2::Mode::Read); + engine.Close(); + } +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 9d87fd637125e0baeef463b5767800bcf40f7c42 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 31 May 2019 11:25:45 -0400 Subject: [PATCH 9/9] bpls: works w/o MPI_Init --- source/utils/bpls/bpls.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 23ee1ffa7e..a70a58f0d4 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1181,11 +1181,7 @@ int doList(const char *path) if (hidden_attrs) strcat(init_params, ";show_hidden_attrs"); -#ifdef ADIOS2_HAVE_MPI - core::ADIOS adios(MPI_COMM_SELF, true, "C++"); -#else core::ADIOS adios(true, "C++"); -#endif core::IO &io = adios.DeclareIO("bpls"); core::Engine *fp = nullptr; std::vector engineList = getEnginesList(path); @@ -2931,9 +2927,6 @@ char *mystrndup(const char *s, size_t n) int main(int argc, char *argv[]) { -#ifdef ADIOS2_HAVE_MPI - MPI_Init(&argc, &argv); -#endif int retval = 1; try { @@ -2944,8 +2937,5 @@ int main(int argc, char *argv[]) std::cout << "\nbpls caught an exception\n"; std::cout << e.what() << std::endl; } -#ifdef ADIOS2_HAVE_MPI - MPI_Finalize(); -#endif return retval; }