Skip to content

Commit 764e400

Browse files
jsquyresjsquyres
authored
Merge pull request #7245 from jsquyres/pr/v3.0.x/neighbor-alltoall-fix
v3.0.x: neighbor alltoall fix
2 parents f2d62b0 + 4e8d84b commit 764e400

File tree

6 files changed

+35
-25
lines changed

6 files changed

+35
-25
lines changed

‎ompi/mca/coll/base/coll_tags.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
#define MCA_COLL_BASE_TAG_SCATTERV -25
4343
#define MCA_COLL_BASE_TAG_NONBLOCKING_BASE -26
4444
#define MCA_COLL_BASE_TAG_NONBLOCKING_END ((-1 * INT_MAX/2) + 1)
45+
#define MCA_COLL_BASE_TAG_NEIGHBOR_BASE (MCA_COLL_BASE_TAG_NONBLOCKING_END - 1)
46+
#define MCA_COLL_BASE_TAG_NEIGHBOR_END (MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 1024)
4547
#define MCA_COLL_BASE_TAG_HCOLL_BASE (-1 * INT_MAX/2)
4648
#define MCA_COLL_BASE_TAG_HCOLL_END (-1 * INT_MAX)
4749
#endif /* MCA_COLL_BASE_TAGS_H */

‎ompi/mca/coll/basic/coll_basic_neighbor_allgather.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -69,15 +69,15 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
6969
if (MPI_PROC_NULL != srank) {
7070
nreqs++;
7171
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
72-
MCA_COLL_BASE_TAG_ALLGATHER,
72+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7373
comm, preqs++));
7474
if (OMPI_SUCCESS != rc) break;
7575

7676
nreqs++;
7777
/* remove cast from const when the pml layer is updated to take
7878
* a const for the send buffer. */
7979
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
80-
MCA_COLL_BASE_TAG_ALLGATHER,
80+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8181
MCA_PML_BASE_SEND_STANDARD,
8282
comm, preqs++));
8383
if (OMPI_SUCCESS != rc) break;
@@ -88,13 +88,13 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
8888
if (MPI_PROC_NULL != drank) {
8989
nreqs++;
9090
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
91-
MCA_COLL_BASE_TAG_ALLGATHER,
91+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
9292
comm, preqs++));
9393
if (OMPI_SUCCESS != rc) break;
9494

9595
nreqs++;
9696
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
97-
MCA_COLL_BASE_TAG_ALLGATHER,
97+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9898
MCA_PML_BASE_SEND_STANDARD,
9999
comm, preqs++));
100100
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -68,25 +68,27 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
6868
if (MPI_PROC_NULL != srank) {
6969
nreqs++;
7070
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i] * extent, rcounts[i], rdtype, srank,
71-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
71+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7272
if (OMPI_SUCCESS != rc) break;
7373

7474
/* remove cast from const when the pml layer is updated to take
7575
* a const for the send buffer. */
7676
nreqs++;
77-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER,
77+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
78+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
7879
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
7980
if (OMPI_SUCCESS != rc) break;
8081
}
8182

8283
if (MPI_PROC_NULL != drank) {
8384
nreqs++;
8485
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i+1] * extent, rcounts[i+1], rdtype, drank,
85-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
86+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
8687
if (OMPI_SUCCESS != rc) break;
8788

8889
nreqs++;
89-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER,
90+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
91+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9092
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9193
if (OMPI_SUCCESS != rc) break;
9294
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,6 +14,8 @@
1414
* reserved.
1515
* Copyright (c) 2014-2015 Research Organization for Information Science
1616
* and Technology (RIST). All rights reserved.
17+
* Copyright (c) 2017 IBM Corporation. All rights reserved.
18+
* Copyright (c) 2019 Google, LLC. All rights reserved.
1719
* $COPYRIGHT$
1820
*
1921
* Additional copyrights may follow
@@ -35,6 +37,10 @@
3537
#include "coll_basic.h"
3638
#include "ompi/mca/topo/base/base.h"
3739

40+
/**
41+
* We only have 1024 tags for the neighbor collective, so for now we only support
42+
* 512 dimensions.
43+
*/
3844
static int
3945
mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
4046
int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
@@ -66,7 +72,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
6672
if (MPI_PROC_NULL != srank) {
6773
nreqs++;
6874
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
69-
MCA_COLL_BASE_TAG_ALLTOALL,
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7076
comm, preqs++));
7177
if (OMPI_SUCCESS != rc) break;
7278
}
@@ -76,7 +82,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
7682
if (MPI_PROC_NULL != drank) {
7783
nreqs++;
7884
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
79-
MCA_COLL_BASE_TAG_ALLTOALL,
85+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8086
comm, preqs++));
8187
if (OMPI_SUCCESS != rc) break;
8288
}
@@ -103,7 +109,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
103109
* a const for the send buffer. */
104110
nreqs++;
105111
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
106-
MCA_COLL_BASE_TAG_ALLTOALL,
112+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
107113
MCA_PML_BASE_SEND_STANDARD,
108114
comm, preqs++));
109115
if (OMPI_SUCCESS != rc) break;
@@ -114,7 +120,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
114120
if (MPI_PROC_NULL != drank) {
115121
nreqs++;
116122
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
117-
MCA_COLL_BASE_TAG_ALLTOALL,
123+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
118124
MCA_PML_BASE_SEND_STANDARD,
119125
comm, preqs++));
120126
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -67,14 +67,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
6767
if (MPI_PROC_NULL != srank) {
6868
nreqs++;
6969
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank,
70-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
70+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7171
if (OMPI_SUCCESS != rc) break;
7272
}
7373

7474
if (MPI_PROC_NULL != drank) {
7575
nreqs++;
7676
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank,
77-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
77+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7878
if (OMPI_SUCCESS != rc) break;
7979
}
8080
}
@@ -97,14 +97,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
9797
nreqs++;
9898
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
9999
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank,
100-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
100+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
101101
if (OMPI_SUCCESS != rc) break;
102102
}
103103

104104
if (MPI_PROC_NULL != drank) {
105105
nreqs++;
106106
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank,
107-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
107+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
108108
if (OMPI_SUCCESS != rc) break;
109109
}
110110
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -65,14 +65,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
6565
if (MPI_PROC_NULL != srank) {
6666
nreqs++;
6767
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i], srank,
68-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
68+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
6969
if (OMPI_SUCCESS != rc) break;
7070
}
7171

7272
if (MPI_PROC_NULL != drank) {
7373
nreqs++;
7474
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1], rcounts[i+1], rdtypes[i+1], drank,
75-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7676
if (OMPI_SUCCESS != rc) break;
7777
}
7878
}
@@ -95,14 +95,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
9595
nreqs++;
9696
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
9797
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i], scounts[i], sdtypes[i], srank,
98-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
98+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9999
if (OMPI_SUCCESS != rc) break;
100100
}
101101

102102
if (MPI_PROC_NULL != drank) {
103103
nreqs++;
104104
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1], scounts[i+1], sdtypes[i+1], drank,
105-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
105+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
106106
if (OMPI_SUCCESS != rc) break;
107107
}
108108
}

0 commit comments

Comments
 (0)