diff --git a/.travis.yml b/.travis.yml index 514561391..75e9d13a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,13 @@ cache: env: - MPI_IMPL=mpich - MPI_IMPL=openmpi + JULIA_MPIEXEC_ARGS="--oversubscribe" - MPI_IMPL=intelmpi + - MPI_IMPL=mpich + JULIA_MPI_ABI=UnknownABI + - MPI_IMPL=openmpi + JULIA_MPI_ABI=UnknownABI + JULIA_MPIEXEC_ARGS="--oversubscribe" matrix: allow_failures: # issue 262 diff --git a/deps/build.jl b/deps/build.jl index e074efd1a..7e2baac25 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -1,5 +1,14 @@ -using Libdl +_doc_external(fname) = "" +include(joinpath("..","src","paths.jl")) +include(joinpath("..","src","implementations.jl")) + +if MPI_LIBRARY_ABI != UnknownABI + # constants provided under src/consts/ + exit(0) +end + +@info "Uknown MPI ABI: building constants file" MPI_PATH = get(ENV, "JULIA_MPI_PATH", nothing) MPI_LIBRARY_PATH = get(ENV, "JULIA_MPI_LIBRARY_PATH") do @@ -8,6 +17,7 @@ end MPI_INCLUDE_PATH = get(ENV, "JULIA_MPI_INCLUDE_PATH") do MPI_PATH !== nothing ? joinpath(MPI_PATH,"include") : nothing end + mpicc = get(ENV, "JULIA_MPICC") do if MPI_PATH !== nothing joinpath(MPI_PATH,"bin","mpicc") @@ -16,27 +26,6 @@ mpicc = get(ENV, "JULIA_MPICC") do end end -const mpiexec_path = get(ENV, "JULIA_MPIEXEC") do - if MPI_PATH !== nothing && Sys.isexecutable(joinpath(MPI_PATH,"bin","mpiexec")) - joinpath(MPI_PATH,"bin","mpiexec") - else - Sys.which("mpiexec") - end -end - -const mpiexec_args = split(get(ENV, "JULIA_MPIEXEC_ARGS", "")) - -const libmpi = get(ENV, "JULIA_MPI_LIBRARY") do - libmpi = find_library(["libmpi", "libmpi_ibm", "msmpi", "libmpich"], - MPI_LIBRARY_PATH !== nothing ? [MPI_LIBRARY_PATH] : []) - if libmpi == "" - error("No MPI library found.\nEnsure an MPI implementation is loaded, or set the `JULIA_MPI_PATH` variable.") - end - # expand paths - dlpath(libmpi) -end - - if haskey(ENV, "JULIA_MPI_CFLAGS") CFLAGS = split(ENV["JULIA_MPI_CFLAGS"]) else @@ -53,65 +42,17 @@ else end end -libsize = filesize(libmpi) - -@info "Using MPI library $libmpi" - -function Get_version() - major = Ref{Cint}() - minor = Ref{Cint}() - if Sys.iswindows() - ccall((:MPI_Get_version, libmpi), stdcall, Cint, - (Ptr{Cint}, Ptr{Cint}), major, minor) - else - ccall((:MPI_Get_version, libmpi), Cint, - (Ptr{Cint}, Ptr{Cint}), major, minor) - end - VersionNumber(major[], minor[]) -end - -MPI_VERSION = Get_version() - -@info "MPI version:\n$(MPI_VERSION)" - -function Get_library_version() - # There is no way to query at runtime what the length of the buffer should be. - # https://github.com/mpi-forum/mpi-issues/issues/159 - # 8192 is the maximum value of MPI_MAX_LIBRARY_VERSION_STRING across known - # implementations. - buf = Array{UInt8}(undef, 8192) - buflen = Ref{Cint}() - if Sys.iswindows() - ccall((:MPI_Get_library_version, libmpi), stdcall, Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen) - else - ccall((:MPI_Get_library_version, libmpi), Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen) - end - resize!(buf, buflen[]) - return String(buf) -end +include("gen_consts.jl") -MPI_LIBRARY_VERSION_STRING = Get_library_version() +run(`$mpicc gen_consts.c -o gen_consts $CFLAGS`) -@info "MPI library version:\n$(MPI_LIBRARY_VERSION_STRING)" - -open("deps.jl","w") do f +open("consts.jl","w") do f println(f, "# This file is automatically generated") println(f, "# Do not edit") println(f) - println(f, :(const libmpi = $libmpi)) - println(f, :(const libmpi_size = $libsize)) - println(f, :(const MPI_VERSION = $MPI_VERSION)) - println(f, :(const MPI_LIBRARY_VERSION_STRING = $MPI_LIBRARY_VERSION_STRING)) - println(f, :(const mpiexec_path = $mpiexec_path)) - - if Sys.iswindows() - println(f, :(include("consts_msmpi.jl"))) - else - include("gen_consts.jl") + println(f, :(MPI_LIBRARY_VERSION_STRING == $MPI_LIBRARY_VERSION_STRING || error("MPI library changed, re-run Pkg.build(\"MPI\")"))) + println(f) +end - run(`$mpicc gen_consts.c -o gen_consts $CFLAGS`) - run(`$mpiexec_path $mpiexec_args -n 1 ./gen_consts`) +mpiexec(cmd -> run(`$cmd -n 1 ./gen_consts`)) - println(f, :(include("consts.jl"))) - end -end diff --git a/deps/gen_consts.jl b/deps/gen_consts.jl index 68b3b4661..0131d4e41 100644 --- a/deps/gen_consts.jl +++ b/deps/gen_consts.jl @@ -133,10 +133,8 @@ int main(int argc, char *argv[]) { MPI_Init(&argc, &argv); FILE *fptr; - fptr = fopen("consts.jl", "w"); + fptr = fopen("consts.jl", "a"); - fprintf(fptr, "# This file is automatically generated\\n"); - fprintf(fptr, "# Do not edit\\n"); """) println(f," fprintf(fptr, \"const MPI_Aint = Int%d\\n\", 8*(int)sizeof(MPI_Aint));") @@ -149,17 +147,16 @@ int main(int argc, char *argv[]) { println(f," fprintf(fptr, \"const MPI_Status_Error_offset = %d\\n\", (int)offsetof(MPI_Status, MPI_ERROR));") for (T,constants) in MPI_handle - println(f," fprintf(fptr, \"primitive type $T %d end\\n\", (int)(sizeof($T) * 8));") - T_f2c = T == :MPI_Datatype ? :MPI_Type_f2c : Symbol(T, :_f2c) T_c2f = T == :MPI_Datatype ? :MPI_Type_c2f : Symbol(T, :_c2f) if Libdl.dlsym_e(libptr, T_f2c) == C_NULL - println(f," fprintf(fptr, \"$T(c::Cint) = reinterpret($T,c)\\n\");") + println(f," fprintf(fptr, \"const $T = Cint\\n\");") for constant in constants println(f," fprintf(fptr, \"const $constant = Cint(%i)\\n\", $constant);") end else - println(f," fprintf(fptr, \"$T(c::Cint) = ccall((:$T_f2c,libmpi),$T,(Cint,),c)\\n\");") + println(f," fprintf(fptr, \"const $T = Ptr{Cvoid}\\n\");") + println(f," fprintf(fptr, \"$(Symbol(T,:_f2c))(c::Cint) = ccall((:$T_f2c,libmpi),$T,(Cint,),c)\\n\");") for constant in constants println(f," fprintf(fptr, \"const $constant = Cint(%i)\\n\", $T_c2f($constant));") end diff --git a/docs/make.jl b/docs/make.jl index b9bce4a12..de79826a0 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -46,6 +46,7 @@ makedocs( "usage.md", "Examples" => EXAMPLES, "Reference" => [ + "library.md", "environment.md", "comm.md", "pointtopoint.md", diff --git a/docs/src/environment.md b/docs/src/environment.md index 585c7d8d4..1d9eeec8c 100644 --- a/docs/src/environment.md +++ b/docs/src/environment.md @@ -1,5 +1,11 @@ # Environment +## Launching MPI programs + +```@docs +mpiexec +``` + ## Functions ```@docs @@ -9,5 +15,4 @@ MPI.Initialized MPI.Finalize MPI.Finalized MPI.universe_size -MPI.has_cuda ``` diff --git a/docs/src/installation.md b/docs/src/installation.md index a799f17a5..d891d9cb8 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -2,30 +2,33 @@ ## Requirements -### Unix systems (Linux and MacOS) +MPI.jl requires a shared library installation of a C MPI library, supporting the MPI 3.0 standard or later. -MPI.jl requires: +## Building + +The MPI.jl package can be installed via `add MPI` in the [Julia package +manager](https://docs.julialang.org/en/v1/stdlib/Pkg/index.html). The package will attempt +to find and identify the MPI installation. -- A shared library MPI installation for C (supporting MPI standard 3.0), and -- A C compiler available via the `mpicc` command: this is required as part of the build - process to determine the necessary type definitions and constants. +The MPI standard doesn't specify the exact application binary interface (ABI), but the +following implementations should work directly: -This has been tested with: - [Open MPI](http://www.open-mpi.org/) -- [MPICH](http://www.mpich.org/) +- [MPICH](http://www.mpich.org/) (v3.1 or later) - [Intel MPI](https://software.intel.com/en-us/mpi-library) +- [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) +- [IBM Spectrum MPI](https://www.ibm.com/us-en/marketplace/spectrum-mpi) -### Windows - -MPI.jl requires the [Microsoft MPI (MS-MPI)](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) runtime to be installed. +For other implementations, the build script will attempt to build a small C program to +determine the appropriate type definitions and constants. This requires a compatible C +compiler (`mpicc` by default). -## Building - -The MPI.jl package can be installed via `add MPI` in the [Julia package manager](https://docs.julialang.org/en/v1/stdlib/Pkg/index.html). +## Environment variables -The build script will attempt to find the shared library and constants: this can be -controlled with the optional environment variables: +The following optional environment variables can be used to control certain aspects of the +build script and other library behaviour: +- `JULIA_MPI_ABI`: the application binary interface, a string matching an [`MPIABI`](@ref) value. - `JULIA_MPI_PATH`: the top-level installation directory of MPI. - `JULIA_MPI_LIBRARY`: the path of the MPI shared library. - `JULIA_MPI_LIBRARY_PATH`: the directory containing the MPI library files. @@ -33,8 +36,5 @@ controlled with the optional environment variables: - `JULIA_MPI_CFLAGS`: C flags passed to the constant generation build (default: `-lmpi`) - `JULIA_MPICC`: MPI C compiler (default: `mpicc`) - `JULIA_MPIEXEC`: MPI launcher command (default: `mpiexec`) -- `JULIA_MPIEXEC_ARGS`: Additional arguments to be passed to MPI launcher (only used in the build step and tests). +- `JULIA_MPIEXEC_ARGS`: Additional arguments to be passed to MPI launcher. - `JULIA_MPI_HAS_CUDA`: override the [`MPI.has_cuda`](@ref) function. - -If your MPI installation changes (e.g. it is upgraded by the system, or you switch -libraries), you will need to re-run `build MPI` at the package prompt. diff --git a/docs/src/library.md b/docs/src/library.md new file mode 100644 index 000000000..e86ba6a60 --- /dev/null +++ b/docs/src/library.md @@ -0,0 +1,24 @@ +# Library information + +## Constants + +```@docs +MPI.MPI_VERSION +MPI.MPI_LIBRARY +MPI.MPI_LIBRARY_VERSION +MPI.MPI_LIBRARY_ABI +MPI.MPI_LIBRARY_VERSION_STRING +``` + +## Enums + +```@docs +MPI.MPIImpl +MPI.MPIABI +``` + +## Functions + +```@docs +MPI.has_cuda +``` diff --git a/docs/src/usage.md b/docs/src/usage.md index 0c51cd5e9..0f266830c 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -26,6 +26,8 @@ Hello world, I am rank 2 of 3 Hello world, I am rank 1 of 3 ``` +The [`mpiexec`](@ref) function is provided for launching MPI programs from Julia itself. + ## CUDA-aware MPI support If your MPI implementation has been compiled with CUDA support, then `CuArray`s (from the diff --git a/src/MPI.jl b/src/MPI.jl index 2a5ad5f0b..6f54a97aa 100644 --- a/src/MPI.jl +++ b/src/MPI.jl @@ -4,6 +4,7 @@ using Libdl, Serialization using Requires using DocStringExtensions +export mpiexec function serialize(x) s = IOBuffer() @@ -31,7 +32,19 @@ function _doc_external(fname) """ end -include(joinpath(@__DIR__, "..", "deps", "deps.jl")) +include("paths.jl") +include("implementations.jl") + +if MPI_LIBRARY_ABI == UnknownABI + include(joinpath(@__DIR__, "..", "deps", "consts.jl")) +elseif MPI_LIBRARY_ABI == MPICHABI + include(joinpath("consts", "mpich.jl")) +elseif MPI_LIBRARY_ABI == OpenMPIABI + include(joinpath("consts", "openmpi.jl")) +elseif MPI_LIBRARY_ABI == MicrosoftMPIABI + include(joinpath("consts", "microsoftmpi.jl")) +end + include("error.jl") include("handle.jl") include("info.jl") @@ -49,6 +62,7 @@ include("io.jl") include("deprecated.jl") function __init__() + @static if Sys.isunix() # need to open libmpi with RTLD_GLOBAL flag for Linux, before # any ccall cannot use RTLD_DEEPBIND; this leads to segfaults @@ -56,6 +70,15 @@ function __init__() Libdl.dlopen(libmpi, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL) end + if MPI_LIBRARY_VERSION_STRING != Get_library_version() + # MPI library has changed, invalidate cache + cachefile = Base.compilecache(Base.PkgId(MPI)) + rm(cachefile) + # TODO: figure out if we can reload package without erroring + # though that would probably trigger a race condition + error("MPI library has changed, please restart Julia") + end + # disable UCX memory hooks since it can mess up dlopen # https://github.com/openucx/ucx/issues/4001 ENV["UCX_MEM_MMAP_RELOC"] = "no" @@ -63,10 +86,6 @@ function __init__() ENV["UCX_MEM_MALLOC_RELOC"] = "no" ENV["UCX_MEM_EVENTS"] = "no" - if filesize(dlpath(libmpi)) != libmpi_size - error("MPI library has changed, re-run Pkg.build(\"MPI\")") - end - @require CuArrays="3a865a2d-5b23-5a0f-bc46-62713ec82fae" include("cuda.jl") end diff --git a/deps/consts_msmpi.jl b/src/consts/microsoftmpi.jl similarity index 93% rename from deps/consts_msmpi.jl rename to src/consts/microsoftmpi.jl index 5df375f3d..7c88ca52b 100644 --- a/deps/consts_msmpi.jl +++ b/src/consts/microsoftmpi.jl @@ -1,24 +1,19 @@ -# From https://github.com/microsoft/Microsoft-MPI/blob/v10.0/src/include/mpi.h - const MPI_Aint = Int const MPI_Offset = Int64 const MPI_Count = Int64 -for T in [:MPI_Comm, :MPI_Info, :MPI_Win, :MPI_Request, :MPI_Op, :MPI_Datatype, :MPI_File] - @eval begin - primitive type $T 32 end - $T(c::Cint) = reinterpret($T, c) - end -end - +const MPI_Comm = Cint const MPI_COMM_NULL = reinterpret(Cint, 0x04000000) const MPI_COMM_SELF = reinterpret(Cint, 0x44000001) const MPI_COMM_WORLD = reinterpret(Cint, 0x44000000) +const MPI_Info = Cint const MPI_INFO_NULL = reinterpret(Cint, 0x1c000000) +const MPI_Win = Cint const MPI_WIN_NULL = reinterpret(Cint, 0x20000000) +const MPI_Op = Cint const MPI_OP_NULL = reinterpret(Cint, 0x18000000) const MPI_MAX = reinterpret(Cint, 0x58000001) const MPI_MIN = reinterpret(Cint, 0x58000002) @@ -35,8 +30,10 @@ const MPI_MAXLOC = reinterpret(Cint, 0x5800000c) const MPI_REPLACE = reinterpret(Cint, 0x5800000d) const MPI_NO_OP = reinterpret(Cint, 0x5800000e) +const MPI_Request = Cint const MPI_REQUEST_NULL = reinterpret(Cint, 0x2c000000) +const MPI_Datatype = Cint const MPI_DATATYPE_NULL = reinterpret(Cint, 0x0c000000) const MPI_PACKED = reinterpret(Cint, 0x4c00010f) const MPI_CHAR = reinterpret(Cint, 0x4c000101) @@ -64,6 +61,7 @@ const MPI_UINT64_T = reinterpret(Cint, 0x4c00083a) const MPI_C_FLOAT_COMPLEX = reinterpret(Cint, 0x4c000813) const MPI_C_DOUBLE_COMPLEX = reinterpret(Cint, 0x4c001014) +const MPI_File = Cint const MPI_FILE_NULL = Cint(0) const MPI_PROC_NULL = Cint(-1) @@ -105,4 +103,5 @@ struct Status tag::Cint error::Cint end + const STATUS_EMPTY = Status(zero(Cint), zero(Cint), MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_SUCCESS) diff --git a/src/consts/mpich.jl b/src/consts/mpich.jl new file mode 100644 index 000000000..64dab1204 --- /dev/null +++ b/src/consts/mpich.jl @@ -0,0 +1,101 @@ +const MPI_Aint = Int +const MPI_Count = Int64 +const MPI_Offset = Int64 + +const MPI_Status_size = 20 +const MPI_Status_Source_offset = 8 +const MPI_Status_Tag_offset = 12 +const MPI_Status_Error_offset = 16 + +const MPI_Info = Cint +const MPI_INFO_NULL = Cint(469762048) + +const MPI_Win = Cint +const MPI_WIN_NULL = Cint(536870912) + +const MPI_Comm = Cint +const MPI_COMM_NULL = Cint(67108864) +const MPI_COMM_SELF = Cint(1140850689) +const MPI_COMM_WORLD = Cint(1140850688) + +const MPI_Request = Cint +const MPI_REQUEST_NULL = Cint(738197504) + +const MPI_File = Ptr{Cvoid} +MPI_File_f2c(c::Cint) = ccall((:MPI_File_f2c,libmpi),MPI_File,(Cint,),c) +const MPI_FILE_NULL = Cint(0) + +const MPI_Op = Cint +const MPI_OP_NULL = Cint(402653184) +const MPI_BAND = Cint(1476395014) +const MPI_BOR = Cint(1476395016) +const MPI_BXOR = Cint(1476395018) +const MPI_LAND = Cint(1476395013) +const MPI_LOR = Cint(1476395015) +const MPI_LXOR = Cint(1476395017) +const MPI_MAX = Cint(1476395009) +const MPI_MIN = Cint(1476395010) +const MPI_SUM = Cint(1476395011) +const MPI_PROD = Cint(1476395012) +const MPI_MAXLOC = Cint(1476395020) +const MPI_MINLOC = Cint(1476395019) +const MPI_REPLACE = Cint(1476395021) +const MPI_NO_OP = Cint(1476395022) + +const MPI_Datatype = Cint +const MPI_DATATYPE_NULL = Cint(201326592) +const MPI_PACKED = Cint(1275068687) +const MPI_CHAR = Cint(1275068673) +const MPI_SIGNED_CHAR = Cint(1275068696) +const MPI_UNSIGNED_CHAR = Cint(1275068674) +const MPI_WCHAR = Cint(1275069454) +const MPI_BYTE = Cint(1275068685) +const MPI_SHORT = Cint(1275068931) +const MPI_UNSIGNED_SHORT = Cint(1275068932) +const MPI_INT = Cint(1275069445) +const MPI_UNSIGNED = Cint(1275069446) +const MPI_LONG = Cint(1275070471) +const MPI_UNSIGNED_LONG = Cint(1275070472) +const MPI_FLOAT = Cint(1275069450) +const MPI_DOUBLE = Cint(1275070475) +const MPI_INT8_T = Cint(1275068727) +const MPI_INT16_T = Cint(1275068984) +const MPI_INT32_T = Cint(1275069497) +const MPI_INT64_T = Cint(1275070522) +const MPI_UINT8_T = Cint(1275068731) +const MPI_UINT16_T = Cint(1275068988) +const MPI_UINT32_T = Cint(1275069501) +const MPI_UINT64_T = Cint(1275070526) +const MPI_C_FLOAT_COMPLEX = Cint(1275070528) +const MPI_C_DOUBLE_COMPLEX = Cint(1275072577) + +const MPI_PROC_NULL = Cint(-1) +const MPI_ANY_SOURCE = Cint(-2) +const MPI_ANY_TAG = Cint(-1) +const MPI_SUCCESS = Cint(0) +const MPI_UNDEFINED = Cint(-32766) +const MPI_LOCK_EXCLUSIVE = Cint(234) +const MPI_LOCK_SHARED = Cint(235) +const MPI_MAX_INFO_KEY = Cint(255) +const MPI_MAX_INFO_VAL = Cint(1024) +const MPI_TAG_UB = Cint(1681915905) +const MPI_COMM_TYPE_SHARED = Cint(1) +const MPI_ORDER_C = Cint(56) +const MPI_ORDER_FORTRAN = Cint(57) +const MPI_UNIVERSE_SIZE = Cint(1681915913) +const MPI_MAX_ERROR_STRING = Cint(512) +const MPI_SUCCESS = Cint(0) +const MPI_MODE_RDONLY = Cint(2) +const MPI_MODE_RDWR = Cint(8) +const MPI_MODE_WRONLY = Cint(4) +const MPI_MODE_CREATE = Cint(1) +const MPI_MODE_EXCL = Cint(64) +const MPI_MODE_DELETE_ON_CLOSE = Cint(16) +const MPI_MODE_UNIQUE_OPEN = Cint(32) +const MPI_MODE_SEQUENTIAL = Cint(256) +const MPI_MODE_APPEND = Cint(128) + +const MPI_BOTTOM = reinterpret(SentinelPtr, 0) +const MPI_IN_PLACE = reinterpret(SentinelPtr, -1) +const MPI_STATUS_IGNORE = reinterpret(SentinelPtr, 1) +const MPI_STATUSES_IGNORE = reinterpret(SentinelPtr, 1) diff --git a/src/consts/openmpi.jl b/src/consts/openmpi.jl new file mode 100644 index 000000000..8419bc8cb --- /dev/null +++ b/src/consts/openmpi.jl @@ -0,0 +1,107 @@ +const MPI_Aint = Int +const MPI_Count = Int64 +const MPI_Offset = Int64 + +const MPI_Status_size = 24 +const MPI_Status_Source_offset = 0 +const MPI_Status_Tag_offset = 4 +const MPI_Status_Error_offset = 8 + +const MPI_Info = Ptr{Cvoid} +MPI_Info_f2c(c::Cint) = ccall((:MPI_Info_f2c,libmpi),MPI_Info,(Cint,),c) +const MPI_INFO_NULL = Cint(0) + +const MPI_Win = Ptr{Cvoid} +MPI_Win_f2c(c::Cint) = ccall((:MPI_Win_f2c,libmpi),MPI_Win,(Cint,),c) +const MPI_WIN_NULL = Cint(0) + +const MPI_Comm = Ptr{Cvoid} +MPI_Comm_f2c(c::Cint) = ccall((:MPI_Comm_f2c,libmpi),MPI_Comm,(Cint,),c) +const MPI_COMM_NULL = Cint(2) +const MPI_COMM_SELF = Cint(1) +const MPI_COMM_WORLD = Cint(0) + +const MPI_Request = Ptr{Cvoid} +MPI_Request_f2c(c::Cint) = ccall((:MPI_Request_f2c,libmpi),MPI_Request,(Cint,),c) +const MPI_REQUEST_NULL = Cint(0) + +const MPI_File = Ptr{Cvoid} +MPI_File_f2c(c::Cint) = ccall((:MPI_File_f2c,libmpi),MPI_File,(Cint,),c) +const MPI_FILE_NULL = Cint(0) + +const MPI_Op = Ptr{Cvoid} +MPI_Op_f2c(c::Cint) = ccall((:MPI_Op_f2c,libmpi),MPI_Op,(Cint,),c) +const MPI_OP_NULL = Cint(0) +const MPI_BAND = Cint(6) +const MPI_BOR = Cint(8) +const MPI_BXOR = Cint(10) +const MPI_LAND = Cint(5) +const MPI_LOR = Cint(7) +const MPI_LXOR = Cint(9) +const MPI_MAX = Cint(1) +const MPI_MIN = Cint(2) +const MPI_SUM = Cint(3) +const MPI_PROD = Cint(4) +const MPI_MAXLOC = Cint(11) +const MPI_MINLOC = Cint(12) +const MPI_REPLACE = Cint(13) +const MPI_NO_OP = Cint(14) + +const MPI_Datatype = Ptr{Cvoid} +MPI_Datatype_f2c(c::Cint) = ccall((:MPI_Type_f2c,libmpi),MPI_Datatype,(Cint,),c) +const MPI_DATATYPE_NULL = Cint(0) +const MPI_PACKED = Cint(2) +const MPI_CHAR = Cint(34) +const MPI_SIGNED_CHAR = Cint(36) +const MPI_UNSIGNED_CHAR = Cint(35) +const MPI_WCHAR = Cint(33) +const MPI_BYTE = Cint(1) +const MPI_SHORT = Cint(37) +const MPI_UNSIGNED_SHORT = Cint(38) +const MPI_INT = Cint(39) +const MPI_UNSIGNED = Cint(40) +const MPI_LONG = Cint(41) +const MPI_UNSIGNED_LONG = Cint(42) +const MPI_FLOAT = Cint(45) +const MPI_DOUBLE = Cint(46) +const MPI_INT8_T = Cint(58) +const MPI_INT16_T = Cint(60) +const MPI_INT32_T = Cint(62) +const MPI_INT64_T = Cint(64) +const MPI_UINT8_T = Cint(59) +const MPI_UINT16_T = Cint(61) +const MPI_UINT32_T = Cint(63) +const MPI_UINT64_T = Cint(65) +const MPI_C_FLOAT_COMPLEX = Cint(69) +const MPI_C_DOUBLE_COMPLEX = Cint(70) + +const MPI_PROC_NULL = Cint(-2) +const MPI_ANY_SOURCE = Cint(-1) +const MPI_ANY_TAG = Cint(-1) +const MPI_SUCCESS = Cint(0) +const MPI_UNDEFINED = Cint(-32766) +const MPI_LOCK_EXCLUSIVE = Cint(1) +const MPI_LOCK_SHARED = Cint(2) +const MPI_MAX_INFO_KEY = Cint(36) +const MPI_MAX_INFO_VAL = Cint(256) +const MPI_TAG_UB = Cint(0) +const MPI_COMM_TYPE_SHARED = Cint(0) +const MPI_ORDER_C = Cint(0) +const MPI_ORDER_FORTRAN = Cint(1) +const MPI_UNIVERSE_SIZE = Cint(6) +const MPI_MAX_ERROR_STRING = Cint(256) +const MPI_SUCCESS = Cint(0) +const MPI_MODE_RDONLY = Cint(2) +const MPI_MODE_RDWR = Cint(8) +const MPI_MODE_WRONLY = Cint(4) +const MPI_MODE_CREATE = Cint(1) +const MPI_MODE_EXCL = Cint(64) +const MPI_MODE_DELETE_ON_CLOSE = Cint(16) +const MPI_MODE_UNIQUE_OPEN = Cint(32) +const MPI_MODE_SEQUENTIAL = Cint(256) +const MPI_MODE_APPEND = Cint(128) + +const MPI_BOTTOM = reinterpret(SentinelPtr, 0) +const MPI_IN_PLACE = reinterpret(SentinelPtr, 1) +const MPI_STATUS_IGNORE = reinterpret(SentinelPtr, 0) +const MPI_STATUSES_IGNORE = reinterpret(SentinelPtr, 0) diff --git a/src/deprecated.jl b/src/deprecated.jl index df85152cf..e99a83852 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,5 +1,7 @@ import Base: @deprecate +## deprecated in v0.13 + ## deprecated in v0.12 @deprecate(Cart_get(comm, maxdims), Cart_get(comm), false) @@ -8,95 +10,3 @@ import Base: @deprecate Cart_coords!(comm, rank, coords), false) @deprecate(Cart_coords(comm, maxdims::Integer), Cart_coords(comm), false) - - -## deprecated in v0.11 - -@deprecate Reduce_in_place!(buf, count::Integer, op, root::Integer, comm::Comm) begin - @assert count == length(buf) - Reduce!(buf, op, root, comm) -end false - -@deprecate Scatter_in_place!(buf, count::Integer, root::Integer, comm::Comm) begin - if root == MPI.Comm_rank(comm) - Scatter!(buf, nothing, count, root, comm) - else - Scatter!(nothing, buf, count, root, comm) - end -end false - -@deprecate Scatterv_in_place!(buf, counts::Vector, root::Integer, comm::Comm) begin - if root == MPI.Comm_rank(comm) - Scatterv!(buf, nothing, counts, root, comm) - else - Scatterv!(nothing, buf, counts, root, comm) - end -end false - -@deprecate Gather_in_place!(buf, count::Integer, root::Integer, comm::Comm) begin - if root == MPI.Comm_rank(comm) - Gather!(nothing, buf, count, root, comm) - else - Gather!(buf, nothing, count, root, comm) - end -end false - -@deprecate Gatherv_in_place!(buf, counts::Vector{Cint}, root::Integer, comm::Comm) begin - if root == MPI.Comm_rank(comm) - Gatherv!(nothing, buf, counts, root, comm) - else - Gatherv!(buf, nothing, counts, root, comm) - end -end false - -@deprecate(mpitype(T), Datatype(T), false) - -@deprecate(Type_Create_Subarray(ndims::Integer, sizes::MPIBuffertype{Cint}, subsizes::MPIBuffertype{Cint}, - starts::MPIBuffertype{Cint}, order::Integer, oldtype), - Types.create_subarray(sizes, subsizes, starts, Datatype(oldtype); rowmajor = order == MPI_ORDER_C), false) -@deprecate(Type_Create_Struct(nfields::Integer, blocklengths::MPIBuffertype{Cint}, - displacements::MPIBuffertype{Cptrdiff_t}, types::MPIBuffertype{MPI_Datatype}), - Types.create_struct(blocklengths, displacements, types), false) -@deprecate(Type_Commit!(datatype), Types.commit!(datatype), false) - - -@deprecate(Send(buf, count::Integer, datatype::Datatype, dest::Integer, tag::Integer, comm::Comm), - Send(Buffer(buf, count, datatype), dest, tag, comm), false) -@deprecate(Send(buf::AbstractArray, count::Integer, dest::Integer, tag::Integer, comm::Comm), - Send(view(buf, 1:count), dest, tag, comm), false) -@deprecate(Send(buf::Ref, count::Integer, dest::Integer, tag::Integer, comm::Comm), - Send(buf, dest, tag, comm), false) - -@deprecate(Isend(buf, count::Integer, datatype::Datatype, dest::Integer, tag::Integer, comm::Comm), - Isend(Buffer(buf,count,datatype), dest, tag, comm), false) -@deprecate(Isend(buf::AbstractArray, count::Integer, dest::Integer, tag::Integer, comm::Comm), - Isend(view(buf,1:count), dest, tag, comm), false) -@deprecate(Isend(buf::Ref, count::Integer, dest::Integer, tag::Integer, comm::Comm), - Isend(buf, dest, tag, comm), false) - -@deprecate(Recv!(buf, count::Integer, datatype::Datatype, src::Integer, tag::Integer, comm::Comm), - Recv!(Buffer(buf, count, datatype), src, tag, comm), false) -@deprecate(Recv!(buf::AbstractArray, count::Integer, src::Integer, tag::Integer, comm::Comm), - Recv!(view(buf, 1:count), src, tag, comm), false) -@deprecate(Recv!(buf::Ref, count::Integer, src::Integer, tag::Integer, comm::Comm), - Recv!(buf, src, tag, comm), false) - -@deprecate(Irecv!(buf, count::Integer, datatype::Datatype, src::Integer, tag::Integer, comm::Comm), - Irecv!(Buffer(buf,count,datatype), src, tag, comm), false) -@deprecate(Irecv!(buf::AbstractArray, count::Integer, src::Integer, tag::Integer, comm::Comm), - Irecv!(view(buf,1:count), src, tag, comm), false) -@deprecate(Irecv!(buf::Ref, count::Integer, src::Integer, tag::Integer, comm::Comm), - Irecv!(buf, src, tag, comm), false) - -@deprecate(Sendrecv!(sendbuf, sendcount::Integer, sendtype, dest::Integer, sendtag::Integer, - recvbuf, recvcount::Integer, recvtype, source::Integer, recvtag::Integer, - comm::Comm), - Sendrecv!(Buffer(sendbuf, sendcount, sendtype), dest, sendtag, - Buffer(recvbuf, recvcount, recvtype), source, recvtag, comm), false) -@deprecate(Sendrecv!(sendbuf, sendcount::Integer, dest::Integer, sendtag::Integer, - recvbuf, recvcount::Integer, source::Integer, recvtag::Integer, - comm::Comm), - Sendrecv!(view(sendbuf, 1:sendcount), dest, sendtag, - view(recvbuf, 1:recvcount), source, recvtag, comm), false) - -Base.@deprecate_binding(mpiexec,mpiexec_path,false) diff --git a/src/environment.jl b/src/environment.jl index efd05a733..6e2d0f9d0 100644 --- a/src/environment.jl +++ b/src/environment.jl @@ -143,7 +143,7 @@ function Finalized() end function Wtick() - @static if Sys.iswindows() + @static if MPI_LIBRARY == MicrosoftMPI ccall((:MPI_Wtick, libmpi), stdcall, Cdouble, ()) else ccall((:MPI_Wtick, libmpi), Cdouble, ()) @@ -151,7 +151,7 @@ function Wtick() end function Wtime() - @static if Sys.iswindows() + @static if MPI_LIBRARY == MicrosoftMPI ccall((:MPI_Wtime, libmpi), stdcall, Cdouble, ()) else ccall((:MPI_Wtime, libmpi), Cdouble, ()) @@ -172,14 +172,12 @@ or `false`. function has_cuda() flag = get(ENV, "JULIA_MPI_HAS_CUDA", nothing) if flag === nothing - # Only OpenMPI provides a function to check CUDA support - # - Spectrum MPI is an OpenMPI, but IBM removed the functionality - # check, therefore force true - @static if occursin("IBM Spectrum MPI", MPI_LIBRARY_VERSION_STRING) - return true - elseif startswith(MPI_LIBRARY_VERSION_STRING, "Open MPI") + # Only Open MPI provides a function to check CUDA support + @static if MPI_LIBRARY == OpenMPI # int MPIX_Query_cuda_support(void) return 0 != ccall((:MPIX_Query_cuda_support, libmpi), Cint, ()) + elseif MPI_LIBRARY == IBMSpectrumMPI + return true else return false end diff --git a/src/error.jl b/src/error.jl index 29839347b..0314a9862 100644 --- a/src/error.jl +++ b/src/error.jl @@ -4,7 +4,7 @@ end macro mpichk(expr) @assert expr isa Expr && expr.head == :call && expr.args[1] == :ccall - if Sys.iswindows() + if MPI_LIBRARY == MicrosoftMPI insert!(expr.args, 3, :stdcall) end :((errcode = $(esc(expr))) == MPI_SUCCESS || throw(MPIError(errcode))) diff --git a/src/handle.jl b/src/handle.jl index f683dae05..c202e12b9 100644 --- a/src/handle.jl +++ b/src/handle.jl @@ -6,6 +6,7 @@ macro mpi_handle(def, mpiname=nothing, extrafields...) if mpiname == nothing mpiname = Symbol(:MPI_, name) end + mpiname_f2c = Symbol(mpiname,:_f2c) quote Base.@__doc__ mutable struct $(esc(def)) val::$mpiname @@ -16,9 +17,13 @@ macro mpi_handle(def, mpiname=nothing, extrafields...) # const initializer function $(esc(_name))(ival::Cint, extraargs...) - x = $name(reinterpret($mpiname, sizeof($mpiname) == 4 ? Int32(-1) : Int64(-1)), extraargs...) - push!(mpi_init_hooks, () -> x.val = $mpiname(ival)) - return x + if $mpiname == Cint + return $name(ival, extraargs...) + else + x = $name(C_NULL, extraargs...) + push!(mpi_init_hooks, () -> x.val = $mpiname_f2c(ival)) + return x + end end function Base.:(==)(a::$name, b::$name) diff --git a/src/implementations.jl b/src/implementations.jl new file mode 100644 index 000000000..244025784 --- /dev/null +++ b/src/implementations.jl @@ -0,0 +1,203 @@ + +function Get_library_version() + # There is no way to query at runtime what the length of the buffer should be. + # https://github.com/mpi-forum/mpi-issues/issues/159 + # 8192 is the maximum value of MPI_MAX_LIBRARY_VERSION_STRING across known + # implementations. + buf = Array{UInt8}(undef, 8192) + buflen = Ref{Cint}() + + # Microsoft MPI uses stdcall calling convention + libfilename, = split(basename(libmpi),'.') + if libfilename == "msmpi" + ccall((:MPI_Get_library_version, libmpi), stdcall, Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen) + else + ccall((:MPI_Get_library_version, libmpi), Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen) + end + resize!(buf, buflen[]) + return String(buf) +end + +""" + MPI_LIBRARY_VERSION_STRING :: String + +The full version string provided by the library + +# External links +$(_doc_external("MPI_Get_library_version")) +""" +const MPI_LIBRARY_VERSION_STRING = Get_library_version() + +""" + MPIImpl + +An enum corresponding to known MPI implementations + +- `UnknownMPI`: unable to determine MPI implementation +- `MPICH`: [MPICH](https://www.mpich.org/) +- `OpenMPI`: [Open MPI](https://www.open-mpi.org/) +- `MicrosoftMPI`: [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) +- `IntelMPI`: [Intel MPI](https://software.intel.com/en-us/mpi-library) +- `SpectrimMPI`: [IBM Spectrum MPI](https://www.ibm.com/us-en/marketplace/spectrum-mpi) +- `MVAPICH`: [MVAPICH](http://mvapich.cse.ohio-state.edu/) + +# See also + +- [`MPI_LIBRARY`](@ref) +""" +@enum MPIImpl begin + UnknownMPI + MPICH + OpenMPI + MicrosoftMPI + IntelMPI + IBMSpectrumMPI + MVAPICH +end + +""" + MPIABI + +An enum corresponding to known MPI Application Binary Interfaces (ABI) + +- `UnknownABI`: unable to determine MPI ABI. This +- `MPICHABI`: Compatible with [MPICH ABI Compatibility Initiative](https://www.mpich.org/abi/). +- `OpenMPIABI`: Compatible with Open MPI +- `MicrosftMPIABI`: Compatible with Microsoft MPI +""" +@enum MPIABI begin + UnknownABI + MPICHABI + OpenMPIABI + MicrosoftMPIABI +end + +""" + impl, version, abi = identify_implementation() + +Attempt to identify the MPI implementation based on +[`MPI_LIBRARY_VERSION_STRING`](@ref). Returns a triple of values: + +- `impl`: a value of type [`MPIImpl`](@ref) +- `version`: a `VersionNumber` of the library, or `nothing` if it cannot be determined. +- `abi`: a value of [`MPIABI`](@ref). This can be overridden by the `JULIA_MPI_ABI` environment variable. +""" +function identify_implementation() + impl = UnknownMPI + version = nothing + abi = UnknownABI + + if startswith(MPI_LIBRARY_VERSION_STRING, "MPICH") + impl = MPICH + # "MPICH Version:\t%s\n" / "MPICH2 Version:\t%s\n" + if (m = match(r"^MPICH2? Version:\t(\d+.\d+.\d+\w*)\n", MPI_LIBRARY_VERSION_STRING)) !== nothing + version = VersionNumber(m.captures[1]) + if version >= v"3.1" + abi = MPICHABI + end + end + + elseif startswith(MPI_LIBRARY_VERSION_STRING, "Open MPI") + # Open MPI / Spectrum MPI + impl = occursin("IBM Spectrum MPI", MPI_LIBRARY_VERSION_STRING) ? IBMSpectrumMPI : OpenMPI + + if (m = match(r"^Open MPI v(\d+.\d+.\d+\w*)", MPI_LIBRARY_VERSION_STRING)) !== nothing + version = VersionNumber(m.captures[1]) + end + abi = OpenMPIABI + + elseif startswith(MPI_LIBRARY_VERSION_STRING, "Microsoft MPI") + impl = MicrosoftMPI + # "Microsoft MPI %u.%u.%u.%u%S" + # ignore last 2 (build numbers) + if (m = match(r"^Microsoft MPI v(\d+.\d+)", MPI_LIBRARY_VERSION_STRING)) !== nothing + version = VersionNumber(m.captures[1]) + end + abi = MicrosoftMPIABI + + elseif startswith(MPI_LIBRARY_VERSION_STRING, "Intel") + impl = IntelMPI + + # TODO: figure out how to parse + # "Intel(R) MPI Library 2019 Update 4 for Linux* OS" + if (m = match(r"^Intel\(R\) MPI Library (\d+)", MPI_LIBRARY_VERSION_STRING)) !== nothing + version = VersionNumber(m.captures[1]) + if version > v"2014" + abi = MPICHABI + end + end + + elseif startswith(MPI_LIBRARY_VERSION_STRING, "MVAPICH2") + impl = MVAPICH + # "MVAPICH2 Version :\t%s\n") + if (m = match(r"^MVAPICH2? Version\s*:\t(\S*)\n", MPI_LIBRARY_VERSION_STRING)) !== nothing + version = VersionNumber(m.captures[1]) + if version > v"2" + abi = MPICHABI + end + end + end + + if (abienv = get(ENV, "JULIA_MPI_ABI", nothing)) !== nothing + for inst in instances(MPIABI) + if String(Symbol(inst)) == abienv + abi = inst + end + end + end + + return impl, version, abi +end + +const MPI_LIBRARY, MPI_LIBRARY_VERSION, MPI_LIBRARY_ABI = identify_implementation() + +""" + MPI_LIBRARY :: MPIImpl + +The current MPI implementation: this is determined by + +# See also +- [`MPIImpl`](@ref) +""" +MPI_LIBRARY + +""" + MPI_LIBRARY_VERSION :: VersionNumber + +The version of the MPI library +""" +MPI_LIBRARY_VERSION + +""" + MPI_LIBRARY_ABI :: MPIABI + +The ABI used by the current MPI implementation. + +# See also +- [`MPIABI`](@ref) +""" +MPI_LIBRARY_ABI + + +function Get_version() + major = Ref{Cint}() + minor = Ref{Cint}() + if MPI_LIBRARY == MicrosoftMPI + ccall((:MPI_Get_version, libmpi), stdcall, Cint, + (Ptr{Cint}, Ptr{Cint}), major, minor) + else + ccall((:MPI_Get_version, libmpi), Cint, + (Ptr{Cint}, Ptr{Cint}), major, minor) + end + VersionNumber(major[], minor[]) +end + +""" + MPI_VERSION :: VersionNumber + +The supported version of the MPI standard. + +# External links +$(_doc_external("MPI_Get_version")) +""" +const MPI_VERSION = Get_version() diff --git a/src/operators.jl b/src/operators.jl index c5db1714c..fec95eb7a 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -70,7 +70,7 @@ end function Op(f, T=Any; iscommutative=false) - if Sys.iswindows() && Sys.WORD_SIZE == 32 + if MPI_LIBRARY == MicrosoftMPI && Sys.WORD_SIZE == 32 error("User-defined reduction operators are not supported on 32-bit Windows.\nSee https://github.com/JuliaParallel/MPI.jl/issues/246 for more details.") end w = OpWrapper{typeof(f),T}(f) diff --git a/src/paths.jl b/src/paths.jl new file mode 100644 index 000000000..b956a0d79 --- /dev/null +++ b/src/paths.jl @@ -0,0 +1,67 @@ +using Libdl + +function find_lib() + MPI_LIBRARY_PATH = get(ENV, "JULIA_MPI_LIBRARY_PATH") do + MPI_PATH = get(ENV, "JULIA_MPI_PATH", nothing) + if MPI_PATH !== nothing + joinpath(MPI_PATH,"lib") + else + nothing + end + end + + libmpi = get(ENV, "JULIA_MPI_LIBRARY") do + find_library(["libmpi", "libmpi_ibm", "msmpi", "libmpich"], + MPI_LIBRARY_PATH !== nothing ? [MPI_LIBRARY_PATH] : []) + end + + if libmpi == "" + error("No MPI library found.\nEnsure an MPI implementation is loaded, or set the `JULIA_MPI_PATH` variable.") + end + + return libmpi +end + +const libmpi = find_lib() + + +function find_mpiexec() + get(ENV, "JULIA_MPIEXEC") do + MPI_PATH = get(ENV, "JULIA_MPI_PATH", nothing) + if MPI_PATH !== nothing && Sys.isexecutable(joinpath(MPI_PATH,"bin","mpiexec")) + joinpath(MPI_PATH,"bin","mpiexec") + else + "mpiexec" + end + end +end + +const mpiexec_path = find_mpiexec() + + +""" + mpiexec(fn) + +A wrapper function for the MPI launcher executable. Calls `fn(cmd)`, where `cmd` is a `Cmd` object of the MPI launcher. + +# Environment Variables + +The behaviour of `mpiexec` can be controlled by the following environment variables: + +- `JULIA_MPIEXEC`: the name or path of the launcher executable (set at compile time). +- `JULIA_MPIEXEC_ARGS`: additional arguments that are passed to the launcher. These are space seperated, supporting the same quoting rules as + Julia `Cmd` objects. These can be modified at run time. + +# Usage + +```jldoctest +julia> mpiexec(cmd -> run(`\$cmd -n 3 echo hello world`)); +hello world +hello world +hello world +``` +""" +function mpiexec(fn) + mpiexec_args = Base.shell_split(get(ENV, "JULIA_MPIEXEC_ARGS", "")) + fn(`$mpiexec_path $mpiexec_args`) +end diff --git a/test/runtests.jl b/test/runtests.jl index fc3eac1d9..e0e6a4128 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,21 +26,18 @@ function runtests() testdir = dirname(@__FILE__) istest(f) = endswith(f, ".jl") && startswith(f, "test_") testfiles = sort(filter(istest, readdir(testdir))) - mpiexec_args = split(get(ENV, "JULIA_MPIEXEC_ARGS", "")) - - if !Sys.iswindows() && occursin( "OpenRTE", read(`$mpiexec_path --version`, String)) - push!(mpiexec_args,"--oversubscribe") - end nfail = 0 printstyled("Running MPI.jl tests\n"; color=:white) for f in testfiles coverage_opt = coverage_opts[Base.JLOptions().code_coverage] - if f ∈ singlefiles - run(`$mpiexec_path $mpiexec_args -n 1 $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`) - else - run(`$mpiexec_path $mpiexec_args -n $nprocs $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`) + mpiexec() do cmd + if f ∈ singlefiles + run(`$cmd -n 1 $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`) + else + run(`$cmd -n $nprocs $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`) + end end Base.with_output_color(:green,stdout) do io println(io,"\tSUCCESS: $f") diff --git a/test/test_allreduce.jl b/test/test_allreduce.jl index 520976e1b..e8fa46573 100644 --- a/test/test_allreduce.jl +++ b/test/test_allreduce.jl @@ -13,7 +13,7 @@ MPI.Init() comm_size = MPI.Comm_size(MPI.COMM_WORLD) if ArrayType != Array || - Sys.iswindows() && Sys.WORD_SIZE == 32 || + MPI.MPI_LIBRARY == MPI.MicrosoftMPI && Sys.WORD_SIZE == 32 || Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le || Sys.ARCH === :aarch64 || startswith(string(Sys.ARCH), "arm") operators = [MPI.SUM, +] diff --git a/test/test_reduce.jl b/test/test_reduce.jl index 5180061cf..249477d97 100644 --- a/test/test_reduce.jl +++ b/test/test_reduce.jl @@ -40,7 +40,7 @@ if isroot @test sum_mesg == sz .* mesg end -if ArrayType != Array || Sys.iswindows() && Sys.WORD_SIZE == 32 || +if ArrayType != Array || MPI.MPI_LIBRARY == MPI.MicrosoftMPI && Sys.WORD_SIZE == 32 || Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le || Sys.ARCH === :aarch64 || startswith(string(Sys.ARCH), "arm") operators = [MPI.SUM, +] @@ -102,7 +102,7 @@ end MPI.Barrier( MPI.COMM_WORLD ) -if Sys.iswindows() && Sys.WORD_SIZE == 32 || +if MPI.MPI_LIBRARY == MPI.MicrosoftMPI && Sys.WORD_SIZE == 32 || Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le || Sys.ARCH === :aarch64 || startswith(string(Sys.ARCH), "arm") # Closures are not supported for cfunction