[coll-comm] disable neighbor comm if open-mpi version < 4.1.0

CMakeLists.txt

@@ -201,7 +201,7 @@ if(GINKGO_BUILD_HWLOC AND (MSVC OR WIN32 OR CYGWIN OR APPLE))
 endif()
 
 set(GINKGO_HAVE_GPU_AWARE_MPI OFF)
-set(GINKGO_FORCE_SPMV_BLOCKING_COMM OFF)
+set(GINKGO_HAVE_OPENMPI_PRE_4_1_X OFF)
 if(GINKGO_BUILD_MPI)
     find_package(MPI 3.1 COMPONENTS CXX REQUIRED)
     if(GINKGO_FORCE_GPU_AWARE_MPI)
@@ -226,12 +226,14 @@ if(GINKGO_BUILD_MPI)
         )
         if(NOT valid_openmpi_version)
             message(WARNING
-                "OpenMPI v4.0.x has a bug that forces us to use blocking communication in our distributed "
+                "OpenMPI v4.0.x has several bugs that forces us to use non-optimal communication in our distributed "
                 "matrix class. To enable faster, non-blocking communication, consider updating your OpenMPI version or "
                 "switch to a different vendor.")
-            set(GINKGO_FORCE_SPMV_BLOCKING_COMM ON)
+            set(GINKGO_HAVE_OPENMPI_PRE_4_1_X ON)
         endif()
+        unset(valid_openmpi_version)
     endif()
+    unset(uses_openmpi)
 endif()
 
 # Try to find the third party packages before using our subdirectories

core/CMakeLists.txt

@@ -134,9 +134,13 @@ if(GINKGO_BUILD_MPI)
         distributed/dense_communicator.cpp
         distributed/matrix.cpp
         distributed/partition_helpers.cpp
-        distributed/neighborhood_communicator.cpp
         distributed/vector.cpp
         distributed/preconditioner/schwarz.cpp)
+    if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X)
+        target_sources(${ginkgo_core}
+            PRIVATE
+            distributed/neighborhood_communicator.cpp)
+    endif()
 endif()
 
 # MSVC/shared: make ginkgo be the major library

core/distributed/dense_communicator.cpp

@@ -80,7 +80,7 @@ request DenseCommunicator::i_all_to_all_v(std::shared_ptr<const Executor> exec,
                                           void* recv_buffer,
                                           MPI_Datatype recv_type) const
 {
-#ifdef GINKGO_FORCE_SPMV_BLOCKING_COMM
+#ifdef GINKGO_HAVE_OPENMPI_PRE_4_1_X
     comm_.all_to_all_v(exec, send_buffer, send_sizes_.data(),
                        send_offsets_.data(), send_type, recv_buffer,
                        recv_sizes_.data(), recv_offsets_.data(), recv_type);

core/distributed/matrix.cpp

@@ -406,7 +406,7 @@ mpi::request Matrix<ValueType, LocalIndexType, GlobalIndexType>::communicate(
     auto recv_ptr = use_host_buffer ? host_recv_buffer_->get_values()
                                     : recv_buffer_->get_values();
     exec->synchronize();
-#ifdef GINKGO_FORCE_SPMV_BLOCKING_COMM
+#ifdef GINKGO_HAVE_OPENMPI_PRE_4_1_X
     comm.all_to_all_v(use_host_buffer ? exec->get_master() : exec, send_ptr,
                       send_sizes_.data(), send_offsets_.data(), type.get(),
                       recv_ptr, recv_sizes_.data(), recv_offsets_.data(),

core/test/mpi/distributed/CMakeLists.txt

@@ -1,7 +1,10 @@
 ginkgo_create_test(helpers MPI_SIZE 1)
 ginkgo_create_test(matrix MPI_SIZE 1)
 ginkgo_create_test(dense_communicator MPI_SIZE 6)
-ginkgo_create_test(neighborhood_communicator MPI_SIZE 6)
+
+if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X)
+    ginkgo_create_test(neighborhood_communicator MPI_SIZE 6)
+endif()
 
 add_subdirectory(preconditioner)
 add_subdirectory(solver)

include/ginkgo/config.hpp.in

@@ -108,7 +108,7 @@
 
 /* Do we need to use blocking communication in our SpMV? */
 // clang-format off
-#cmakedefine GINKGO_FORCE_SPMV_BLOCKING_COMM
+#cmakedefine01 GINKGO_HAVE_OPENMPI_PRE_4_1_X
 // clang-format on
 
 

include/ginkgo/core/distributed/neighborhood_communicator.hpp

@@ -9,7 +9,7 @@
 #include <ginkgo/config.hpp>
 
 
-#if GINKGO_BUILD_MPI
+#if GINKGO_BUILD_MPI && !GINKGO_HAVE_OPENMPI_PRE_4_1_X
 
 #include <ginkgo/core/base/mpi.hpp>
 #include <ginkgo/core/distributed/collective_communicator.hpp>