coll/libnbc: do not handle MPI_IN_PLACE in neighborhood collectives

ggouaillardet · ggouaillardet · commit bbfa86535022 · 2016-03-09T15:18:52.000+09:00
MPI_IN_PLACE is not a valid send buffer for neighborhood collectives, so just ignore it here. This commit is a small subset of open-mpi/ompi@d42e096 Thanks Jun Kudo for the report.
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c
@@ -1,9 +1,11 @@
 /*
- * Copyright (c) 2006 The Trustees of Indiana University and Indiana
- *                    University Research and Technology
- *                    Corporation.  All rights reserved.
- * Copyright (c) 2006 The Technical University of Chemnitz. All
- *                    rights reserved.
+ * Copyright (c) 2006      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2006      The Technical University of Chemnitz. All
+ *                         rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
@@ -56,14 +58,11 @@ int ompi_coll_libnbc_ineighbor_allgather(void *sbuf, int scount, MPI_Datatype st
   res = MPI_Type_extent(rtype, &rcvext);
   if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
 
-  char inplace;
   NBC_Schedule *schedule;
 #ifdef NBC_CACHE_SCHEDULE
   NBC_Ineighbor_allgather_args *args, *found, search;
 #endif
 
-  NBC_IN_PLACE(sbuf, rbuf, inplace);
-
   handle->tmpbuf=NULL;
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -93,41 +92,17 @@ int ompi_coll_libnbc_ineighbor_allgather(void *sbuf, int scount, MPI_Datatype st
       res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
       if(res != NBC_OK) return res;
 
-      if(inplace) { /* we need an extra buffer to be deadlock-free */
-        handle->tmpbuf = malloc(indegree*rcvext*rcount);
-
-        for(i = 0; i < indegree; i++) {
-          if (MPI_PROC_NULL != srcs[i]) {
-            res = NBC_Sched_recv((char*)0+i*rcount*rcvext, true, rcount, rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if (MPI_PROC_NULL != dsts[i]) {
-            res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
+      /* simply loop over neighbors and post send/recv operations */
+      for(i = 0; i < indegree; i++) {
+        if (MPI_PROC_NULL != srcs[i]) {
+          res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
         }
-        /* unpack from buffer */
-        for(i = 0; i < indegree; i++) {
-          res = NBC_Sched_barrier(schedule);
-          if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
-          res = NBC_Sched_copy((char*)0+i*rcount*rcvext, true, rcount, rtype, (char*)rbuf+i*rcount*rcvext, false, rcount, rtype, schedule);
-          if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
-        }
-      } else { /* non INPLACE case */
-        /* simply loop over neighbors and post send/recv operations */
-        for(i = 0; i < indegree; i++) {
-          if (MPI_PROC_NULL != srcs[i]) {
-            res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if (MPI_PROC_NULL != dsts[i]) {
-            res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
+      }
+      for(i = 0; i < outdegree; i++) {
+        if (MPI_PROC_NULL != dsts[i]) {
+          res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
         }
       }
     }
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c
@@ -1,9 +1,11 @@
 /*
- * Copyright (c) 2006 The Trustees of Indiana University and Indiana
- *                    University Research and Technology
- *                    Corporation.  All rights reserved.
- * Copyright (c) 2006 The Technical University of Chemnitz. All
- *                    rights reserved.
+ * Copyright (c) 2006      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2006      The Technical University of Chemnitz. All
+ *                         rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
@@ -57,14 +59,11 @@ int ompi_coll_libnbc_ineighbor_allgatherv(void *sbuf, int scount, MPI_Datatype s
   res = MPI_Type_extent(rtype, &rcvext);
   if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
 
-  char inplace;
   NBC_Schedule *schedule;
 #ifdef NBC_CACHE_SCHEDULE
   NBC_Ineighbor_allgatherv_args *args, *found, search;
 #endif
 
-  NBC_IN_PLACE(sbuf, rbuf, inplace);
-
   handle->tmpbuf=NULL;
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -94,50 +93,17 @@ int ompi_coll_libnbc_ineighbor_allgatherv(void *sbuf, int scount, MPI_Datatype s
       res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
       if(res != NBC_OK) return res;
 
-      if(inplace) { /* we need an extra buffer to be deadlock-free */
-        int sumrcounts=0;
-        int offset=0;
-        for(i=0; i<indegree; ++i) sumrcounts += rcounts[i];
-        handle->tmpbuf = malloc(rcvext*sumrcounts);
-
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-          offset += rcounts[i]*rcvext;
-        }
-        for(i = 0; i < outdegree; i++) {
-          if(dsts[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
-        }
-        /* unpack from buffer */
-        offset=0;
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_barrier(schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
-            res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtype, (char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
-          }
-          offset += rcounts[i]*rcvext;
+      /* simply loop over neighbors and post send/recv operations */
+      for(i = 0; i < indegree; i++) {
+        if(srcs[i] != MPI_PROC_NULL) {
+          res = NBC_Sched_recv((char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
         }
-      } else { /* non INPLACE case */
-
-        /* simply loop over neighbors and post send/recv operations */
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_recv((char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if(dsts[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
+      }
+      for(i = 0; i < outdegree; i++) {
+        if(dsts[i] != MPI_PROC_NULL) {
+          res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
         }
       }
     }
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c
@@ -1,9 +1,11 @@
 /*
- * Copyright (c) 2006 The Trustees of Indiana University and Indiana
- *                    University Research and Technology
- *                    Corporation.  All rights reserved.
- * Copyright (c) 2006 The Technical University of Chemnitz. All
- *                    rights reserved.
+ * Copyright (c) 2006      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2006      The Technical University of Chemnitz. All
+ *                         rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
@@ -55,14 +57,11 @@ int ompi_coll_libnbc_ineighbor_alltoall(void *sbuf, int scount, MPI_Datatype sty
   res = MPI_Type_extent(rtype, &rcvext);
   if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
 
-  char inplace;
   NBC_Schedule *schedule;
 #ifdef NBC_CACHE_SCHEDULE
   NBC_Ineighbor_alltoall_args *args, *found, search;
 #endif
 
-  NBC_IN_PLACE(sbuf, rbuf, inplace);
-
   handle->tmpbuf=NULL;
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -92,41 +91,17 @@ int ompi_coll_libnbc_ineighbor_alltoall(void *sbuf, int scount, MPI_Datatype sty
       res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
       if(res != NBC_OK) return res;
 
-      if(inplace) { /* we need an extra buffer to be deadlock-free */
-        handle->tmpbuf = malloc(indegree*rcvext*rcount);
-
-        for(i = 0; i < indegree; i++) {
-          if (MPI_PROC_NULL != srcs[i]) {
-	    res = NBC_Sched_recv((char*)0+i*rcount*rcvext, true, rcount, rtype, srcs[i], schedule);
-	    if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-	  }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if (MPI_PROC_NULL != dsts[i]) {
-	    res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
-	    if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-	  }
+      /* simply loop over neighbors and post send/recv operations */
+      for(i = 0; i < indegree; i++) {
+        if (MPI_PROC_NULL != srcs[i]) {
+          res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
         }
-        /* unpack from buffer */
-        for(i = 0; i < indegree; i++) {
-          res = NBC_Sched_barrier(schedule);
-          if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
-          res = NBC_Sched_copy((char*)0+i*rcount*rcvext, true, rcount, rtype, (char*)rbuf+i*rcount*rcvext, false, rcount, rtype, schedule);
-          if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
-        }
-      } else { /* non INPLACE case */
-        /* simply loop over neighbors and post send/recv operations */
-        for(i = 0; i < indegree; i++) {
-          if (MPI_PROC_NULL != srcs[i]) {
-	    res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
-	    if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-	  }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if (MPI_PROC_NULL != dsts[i]) {
-	    res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
-	    if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-	  }
+      }
+      for(i = 0; i < outdegree; i++) {
+        if (MPI_PROC_NULL != dsts[i]) {
+          res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
         }
       }
     }
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c
@@ -1,9 +1,11 @@
 /*
- * Copyright (c) 2006 The Trustees of Indiana University and Indiana
- *                    University Research and Technology
- *                    Corporation.  All rights reserved.
- * Copyright (c) 2006 The Technical University of Chemnitz. All
- *                    rights reserved.
+ * Copyright (c) 2006      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2006      The Technical University of Chemnitz. All
+ *                         rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
@@ -57,14 +59,11 @@ int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls,
   res = MPI_Type_extent(rtype, &rcvext);
   if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
 
-  char inplace;
   NBC_Schedule *schedule;
 #ifdef NBC_CACHE_SCHEDULE
   NBC_Ineighbor_alltoallv_args *args, *found, search;
 #endif
 
-  NBC_IN_PLACE(sbuf, rbuf, inplace);
-
   handle->tmpbuf=NULL;
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -94,50 +93,17 @@ int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls,
       res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
       if(res != NBC_OK) return res;
 
-      if(inplace) { /* we need an extra buffer to be deadlock-free */
-        int sumrcounts=0;
-        int offset=0;
-        for(i=0; i<indegree; ++i) sumrcounts += rcounts[i];
-        handle->tmpbuf = malloc(rcvext*sumrcounts);
-
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-          offset += rcounts[i]*rcvext;
-        }
-
-        for(i = 0; i < outdegree; i++) {
-          if(dsts[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
+      /* simply loop over neighbors and post send/recv operations */
+      for(i = 0; i < indegree; i++) {
+        if(srcs[i] != MPI_PROC_NULL) {
+          res = NBC_Sched_recv((char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
         }
-        /* unpack from buffer */
-        offset=0;
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_barrier(schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
-            res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtype, (char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
-          }
-          offset += rcounts[i]*rcvext;
-        }
-      } else { /* non INPLACE case */
-        /* simply loop over neighbors and post send/recv operations */
-        for(i = 0; i < indegree; i++) {
-          if(srcs[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_recv((char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
-          }
-        }
-        for(i = 0; i < outdegree; i++) {
-          if(dsts[i] != MPI_PROC_NULL) {
-            res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
-            if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
-          }
+      }
+      for(i = 0; i < outdegree; i++) {
+        if(dsts[i] != MPI_PROC_NULL) {
+          res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
+          if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
         }
       }
     }
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c
