Skip to content

Commit 8b424c3

Browse files
jsquyresjsquyres
authored
Merge pull request #7232 from bosilca/hjelmn_neighbor_alltoall_fix
Neighbor alltoall fix
2 parents 1af6dbe + 86acdee commit 8b424c3

File tree

6 files changed

+34
-25
lines changed

6 files changed

+34
-25
lines changed

‎ompi/mca/coll/base/coll_tags.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
#define MCA_COLL_BASE_TAG_SCATTERV -26
4444
#define MCA_COLL_BASE_TAG_NONBLOCKING_BASE -27
4545
#define MCA_COLL_BASE_TAG_NONBLOCKING_END ((-1 * INT_MAX/2) + 1)
46+
#define MCA_COLL_BASE_TAG_NEIGHBOR_BASE (MCA_COLL_BASE_TAG_NONBLOCKING_END - 1)
47+
#define MCA_COLL_BASE_TAG_NEIGHBOR_END (MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 1024)
4648
#define MCA_COLL_BASE_TAG_HCOLL_BASE (-1 * INT_MAX/2)
4749
#define MCA_COLL_BASE_TAG_HCOLL_END (-1 * INT_MAX)
4850
#endif /* MCA_COLL_BASE_TAGS_H */

‎ompi/mca/coll/basic/coll_basic_neighbor_allgather.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -70,15 +70,15 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
7070
if (MPI_PROC_NULL != srank) {
7171
nreqs++;
7272
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
73-
MCA_COLL_BASE_TAG_ALLGATHER,
73+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7474
comm, preqs++));
7575
if (OMPI_SUCCESS != rc) break;
7676

7777
nreqs++;
7878
/* remove cast from const when the pml layer is updated to take
7979
* a const for the send buffer. */
8080
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
81-
MCA_COLL_BASE_TAG_ALLGATHER,
81+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8282
MCA_PML_BASE_SEND_STANDARD,
8383
comm, preqs++));
8484
if (OMPI_SUCCESS != rc) break;
@@ -89,13 +89,13 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
8989
if (MPI_PROC_NULL != drank) {
9090
nreqs++;
9191
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
92-
MCA_COLL_BASE_TAG_ALLGATHER,
92+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
9393
comm, preqs++));
9494
if (OMPI_SUCCESS != rc) break;
9595

9696
nreqs++;
9797
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
98-
MCA_COLL_BASE_TAG_ALLGATHER,
98+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9999
MCA_PML_BASE_SEND_STANDARD,
100100
comm, preqs++));
101101
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -69,25 +69,27 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
6969
if (MPI_PROC_NULL != srank) {
7070
nreqs++;
7171
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i] * extent, rcounts[i], rdtype, srank,
72-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
72+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7373
if (OMPI_SUCCESS != rc) break;
7474

7575
/* remove cast from const when the pml layer is updated to take
7676
* a const for the send buffer. */
7777
nreqs++;
78-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER,
78+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
79+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
7980
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
8081
if (OMPI_SUCCESS != rc) break;
8182
}
8283

8384
if (MPI_PROC_NULL != drank) {
8485
nreqs++;
8586
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i+1] * extent, rcounts[i+1], rdtype, drank,
86-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
87+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
8788
if (OMPI_SUCCESS != rc) break;
8889

8990
nreqs++;
90-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER,
91+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
92+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9193
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9294
if (OMPI_SUCCESS != rc) break;
9395
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -15,6 +15,7 @@
1515
* Copyright (c) 2014-2015 Research Organization for Information Science
1616
* and Technology (RIST). All rights reserved.
1717
* Copyright (c) 2017 IBM Corporation. All rights reserved.
18+
* Copyright (c) 2019 Google, LLC. All rights reserved.
1819
* $COPYRIGHT$
1920
*
2021
* Additional copyrights may follow
@@ -36,6 +37,10 @@
3637
#include "coll_basic.h"
3738
#include "ompi/mca/topo/base/base.h"
3839

40+
/**
41+
* We only have 1024 tags for the neighbor collective, so for now we only support
42+
* 512 dimensions.
43+
*/
3944
static int
4045
mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
4146
int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
@@ -67,7 +72,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
6772
if (MPI_PROC_NULL != srank) {
6873
nreqs++;
6974
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
70-
MCA_COLL_BASE_TAG_ALLTOALL,
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7176
comm, preqs++));
7277
if (OMPI_SUCCESS != rc) break;
7378
}
@@ -77,7 +82,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
7782
if (MPI_PROC_NULL != drank) {
7883
nreqs++;
7984
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
80-
MCA_COLL_BASE_TAG_ALLTOALL,
85+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8186
comm, preqs++));
8287
if (OMPI_SUCCESS != rc) break;
8388
}
@@ -104,7 +109,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
104109
* a const for the send buffer. */
105110
nreqs++;
106111
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
107-
MCA_COLL_BASE_TAG_ALLTOALL,
112+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
108113
MCA_PML_BASE_SEND_STANDARD,
109114
comm, preqs++));
110115
if (OMPI_SUCCESS != rc) break;
@@ -115,7 +120,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
115120
if (MPI_PROC_NULL != drank) {
116121
nreqs++;
117122
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
118-
MCA_COLL_BASE_TAG_ALLTOALL,
123+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
119124
MCA_PML_BASE_SEND_STANDARD,
120125
comm, preqs++));
121126
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -68,14 +68,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
6868
if (MPI_PROC_NULL != srank) {
6969
nreqs++;
7070
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank,
71-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
71+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7272
if (OMPI_SUCCESS != rc) break;
7373
}
7474

7575
if (MPI_PROC_NULL != drank) {
7676
nreqs++;
7777
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank,
78-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
78+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7979
if (OMPI_SUCCESS != rc) break;
8080
}
8181
}
@@ -98,14 +98,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
9898
nreqs++;
9999
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
100100
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank,
101-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
101+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
102102
if (OMPI_SUCCESS != rc) break;
103103
}
104104

105105
if (MPI_PROC_NULL != drank) {
106106
nreqs++;
107107
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank,
108-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
108+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
109109
if (OMPI_SUCCESS != rc) break;
110110
}
111111
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -65,14 +65,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
6565
if (MPI_PROC_NULL != srank) {
6666
nreqs++;
6767
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i], srank,
68-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
68+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
6969
if (OMPI_SUCCESS != rc) break;
7070
}
7171

7272
if (MPI_PROC_NULL != drank) {
7373
nreqs++;
7474
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1], rcounts[i+1], rdtypes[i+1], drank,
75-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7676
if (OMPI_SUCCESS != rc) break;
7777
}
7878
}
@@ -95,14 +95,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
9595
nreqs++;
9696
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
9797
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i], scounts[i], sdtypes[i], srank,
98-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
98+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9999
if (OMPI_SUCCESS != rc) break;
100100
}
101101

102102
if (MPI_PROC_NULL != drank) {
103103
nreqs++;
104104
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1], scounts[i+1], sdtypes[i+1], drank,
105-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
105+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
106106
if (OMPI_SUCCESS != rc) break;
107107
}
108108
}

0 commit comments

Comments
 (0)