diff --git a/maint/gen_coll.py b/maint/gen_coll.py index afd46f026e6..cf77d6fea5a 100644 --- a/maint/gen_coll.py +++ b/maint/gen_coll.py @@ -165,6 +165,8 @@ def dump_allcomm_sched_auto(name): dump_split(0, "int MPIR_%s_allcomm_sched_auto(%s)" % (Name, func_params)) dump_open('{') G.out.append("int mpi_errno = MPI_SUCCESS;") + if re.match(r'Ineighbor_', Name): + G.out.append("int coll_group = MPIR_SUBGROUP_NONE;") G.out.append("") # -- Csel_search @@ -367,6 +369,8 @@ def dump_cases(commkind): dump_split(0, "int MPIR_%s_sched_impl(%s)" % (Name, func_params)) dump_open('{') G.out.append("int mpi_errno = MPI_SUCCESS;") + if re.match(r'Ineighbor_', Name): + G.out.append("int coll_group = MPIR_SUBGROUP_NONE;") G.out.append("") dump_open("if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) {") diff --git a/src/include/mpir_nbc.h b/src/include/mpir_nbc.h index 9320bf5b633..710521d9a5d 100644 --- a/src/include/mpir_nbc.h +++ b/src/include/mpir_nbc.h @@ -45,7 +45,7 @@ /* Open question: should tag allocation be rolled into Sched_start? Keeping it * separate potentially allows more parallelism in the future, but it also * pushes more work onto the clients of this interface. */ -int MPIR_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag); +int MPIR_Sched_next_tag(MPIR_Comm * comm_ptr, int coll_group, int *tag); void MPIR_Sched_set_tag(MPIR_Sched_t s, int tag); /* the device must provide a typedef for MPIR_Sched_t in mpidpre.h */ diff --git a/src/mpi/coll/allreduce/allreduce_intra_ring.c b/src/mpi/coll/allreduce/allreduce_intra_ring.c index fadd14d6b98..88e2c2e695d 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_ring.c +++ b/src/mpi/coll/allreduce/allreduce_intra_ring.c @@ -74,7 +74,7 @@ int MPIR_Allreduce_intra_ring(const void *sendbuf, void *recvbuf, MPI_Aint count send_rank = (nranks + rank - 1 - i) % nranks; /* get a new tag to prevent out of order messages */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIC_Irecv(tmpbuf, cnts[recv_rank], datatype, src, tag, diff --git a/src/mpi/coll/allreduce/allreduce_intra_tree.c b/src/mpi/coll/allreduce/allreduce_intra_tree.c index 03bcad74cb3..73915a72484 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_tree.c +++ b/src/mpi/coll/allreduce/allreduce_intra_tree.c @@ -139,7 +139,7 @@ int MPIR_Allreduce_intra_tree(const void *sendbuf, void *reduce_address = (char *) reduce_buffer + offset * extent; MPIR_ERR_CHKANDJUMP(!reduce_address, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/iallgather/iallgather_tsp_brucks.c b/src/mpi/coll/iallgather/iallgather_tsp_brucks.c index d30e85ceca8..72ac971069f 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_brucks.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_brucks.c @@ -40,7 +40,7 @@ MPIR_TSP_Iallgather_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_FUNC_ENTER; diff --git a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c index 6663ba9c96c..bb3dfcacdc1 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c @@ -255,7 +255,7 @@ int MPIR_TSP_Iallgather_sched_intra_recexch(const void *sendbuf, MPI_Aint sendco /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); is_inplace = (sendbuf == MPI_IN_PLACE); diff --git a/src/mpi/coll/iallgather/iallgather_tsp_ring.c b/src/mpi/coll/iallgather/iallgather_tsp_ring.c index 700f972a083..7b5e34ce881 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_ring.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_ring.c @@ -83,7 +83,7 @@ int MPIR_TSP_Iallgather_sched_intra_ring(const void *sendbuf, MPI_Aint sendcount int recv_id[3] = { 0 }; /* warning fix: icc: maybe used before set */ for (i = 0; i < size - 1; i++) { /* Get new tag for each cycle so that the send-recv pairs are matched correctly */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int vtcs[3], nvtcs; diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c index 4eb3f04579d..0a72891cf7d 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c @@ -64,7 +64,7 @@ MPIR_TSP_Iallgatherv_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); is_inplace = (sendbuf == MPI_IN_PLACE); diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c index a22694e0c6a..1751ac4c091 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c @@ -273,7 +273,7 @@ int MPIR_TSP_Iallgatherv_sched_intra_recexch(const void *sendbuf, MPI_Aint sendc /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* get the neighbors, the function allocates the required memory */ diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c index bc5cdd96978..d014fa777a2 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c @@ -84,7 +84,7 @@ int MPIR_TSP_Iallgatherv_sched_intra_ring(const void *sendbuf, MPI_Aint sendcoun send_rank = (rank - i + nranks) % nranks; /* Rank whose data you're sending */ /* New tag for each send-recv pair. */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int nvtcs, vtcs[3]; diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c index 85646e3d637..38679c4907a 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c @@ -50,7 +50,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch(const void *sendbuf, void *recvbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); /* get the neighbors, the function allocates the required memory */ MPII_Recexchalgo_get_neighbors(rank, nranks, &k, &step1_sendto, diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c index 284ca9e5b19..2ecde2af9d9 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c @@ -54,7 +54,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(co /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* get the neighbors, the function allocates the required memory */ diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c b/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c index a461635964c..5906a710024 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c @@ -81,7 +81,7 @@ int MPIR_TSP_Iallreduce_sched_intra_ring(const void *sendbuf, void *recvbuf, MPI send_rank = (nranks + rank - 1 - i) % nranks; /* get a new tag to prevent out of order messages */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); nvtcs = (i == 0) ? 0 : 1; diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c b/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c index 4c5d3d79b50..a0ea3ddd7dd 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c @@ -117,7 +117,7 @@ int MPIR_TSP_Iallreduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c b/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c index 9ab18d690e9..aabd475f413 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c @@ -146,7 +146,7 @@ MPIR_TSP_Ialltoall_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_CHKLMEM_MALLOC(pack_invtcs, int *, sizeof(int) * k, mpi_errno, "pack_invtcs", diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c b/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c index 989769feb15..29e34d080d6 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c @@ -117,7 +117,7 @@ int MPIR_TSP_Ialltoall_sched_intra_ring(const void *sendbuf, MPI_Aint sendcount, for (i = 0; i < size - 1; i++) { /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int vtcs[3], nvtcs; diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c b/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c index 1fbf7d2b383..506a5546d5c 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c @@ -58,7 +58,7 @@ int MPIR_TSP_Ialltoall_sched_intra_scattered(const void *sendbuf, MPI_Aint sendc /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, size); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c index 2e3b4d31829..1bb6902f2ba 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c @@ -28,7 +28,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_blocked(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c index 80feead1da9..039e1e4c693 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c @@ -27,7 +27,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_inplace(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c index 57ec5f136d9..8bf0b6cedfa 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c @@ -55,7 +55,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_scattered(const void *sendbuf, const MPI_Ain MPIR_Type_get_true_extent_impl(sendtype, &sendtype_lb, &sendtype_true_extent); sendtype_extent = MPL_MAX(sendtype_extent, sendtype_true_extent); - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* First, post bblock number of sends/recvs */ diff --git a/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c b/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c index 9675c9ffe3b..922a7cd4a00 100644 --- a/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c +++ b/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c @@ -32,7 +32,7 @@ int MPIR_TSP_Ialltoallw_sched_intra_blocked(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* post only bblock isends/irecvs at a time as suggested by Tony Ladd */ diff --git a/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c b/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c index 7af7c386b4b..8bebe075c03 100644 --- a/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c +++ b/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c @@ -31,7 +31,7 @@ int MPIR_TSP_Ialltoallw_sched_intra_inplace(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* FIXME: Here we allocate tmp_buf using extent and send/recv with datatype directly, diff --git a/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c b/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c index e59e162ab65..323c41eb3cb 100644 --- a/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c +++ b/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c @@ -22,7 +22,7 @@ int MPIR_TSP_Ibarrier_sched_intra_k_dissemination(MPIR_Comm * comm, int coll_gro MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c index 5bece20ed41..30cff266f12 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c @@ -32,7 +32,7 @@ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_FUNC_ENTER; diff --git a/src/mpi/coll/ibcast/ibcast_tsp_tree.c b/src/mpi/coll/ibcast/ibcast_tsp_tree.c index 836ffbed01c..4ec7336c063 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_tree.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_tree.c @@ -61,7 +61,7 @@ int MPIR_TSP_Ibcast_sched_intra_tree(void *buffer, MPI_Aint count, MPI_Datatype /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* Receive message from parent */ diff --git a/src/mpi/coll/igather/igather_tsp_tree.c b/src/mpi/coll/igather/igather_tsp_tree.c index 7799da9bdc2..bbb5dee7059 100644 --- a/src/mpi/coll/igather/igather_tsp_tree.c +++ b/src/mpi/coll/igather/igather_tsp_tree.c @@ -45,7 +45,7 @@ int MPIR_TSP_Igather_sched_intra_tree(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); if (rank == root && is_inplace) { diff --git a/src/mpi/coll/igatherv/igatherv_tsp_linear.c b/src/mpi/coll/igatherv/igatherv_tsp_linear.c index 324cebfb441..f3a941a22da 100644 --- a/src/mpi/coll/igatherv/igatherv_tsp_linear.c +++ b/src/mpi/coll/igatherv/igatherv_tsp_linear.c @@ -37,7 +37,7 @@ int MPIR_TSP_Igatherv_sched_allcomm_linear(const void *sendbuf, MPI_Aint sendcou comm_size = comm_ptr->remote_size; } - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* If rank == root, then I recv lots, otherwise I send */ diff --git a/src/mpi/coll/include/coll_impl.h b/src/mpi/coll/include/coll_impl.h index b5b576b1b33..cef96e59aeb 100644 --- a/src/mpi/coll/include/coll_impl.h +++ b/src/mpi/coll/include/coll_impl.h @@ -75,7 +75,7 @@ int MPII_Coll_finalize(void); mpi_errno = MPIR_Sched_create(&s, sched_kind); \ MPIR_ERR_CHECK(mpi_errno); \ int tag = -1; \ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); \ + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); \ MPIR_ERR_CHECK(mpi_errno); \ MPIR_Sched_set_tag(s, tag); \ *sched_type_p = MPIR_SCHED_NORMAL; \ diff --git a/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c b/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c index 6f3218c2c62..c0717d70ddd 100644 --- a/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c +++ b/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c @@ -38,7 +38,7 @@ int MPIR_TSP_Ineighbor_allgather_sched_allcomm_linear(const void *sendbuf, MPI_A /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c b/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c index c8a3d70867e..863b86e4973 100644 --- a/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c +++ b/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c @@ -39,7 +39,7 @@ int MPIR_TSP_Ineighbor_allgatherv_sched_allcomm_linear(const void *sendbuf, MPI_ /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c b/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c index 74e3caa9ca6..4b427bfbe73 100644 --- a/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c @@ -39,7 +39,7 @@ int MPIR_TSP_Ineighbor_alltoall_sched_allcomm_linear(const void *sendbuf, MPI_Ai /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c b/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c index 6ed728dc8a5..87eead1a457 100644 --- a/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c @@ -43,7 +43,7 @@ int MPIR_TSP_Ineighbor_alltoallv_sched_allcomm_linear(const void *sendbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c b/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c index bc6501a51ba..711f95a220d 100644 --- a/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c @@ -39,7 +39,7 @@ int MPIR_TSP_Ineighbor_alltoallw_sched_allcomm_linear(const void *sendbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ireduce/ireduce_tsp_tree.c b/src/mpi/coll/ireduce/ireduce_tsp_tree.c index efa8682bb40..4fbd0fe2ae5 100644 --- a/src/mpi/coll/ireduce/ireduce_tsp_tree.c +++ b/src/mpi/coll/ireduce/ireduce_tsp_tree.c @@ -193,7 +193,7 @@ int MPIR_TSP_Ireduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI_Ai /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c b/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c index f71afdf58cf..1d0e23bf409 100644 --- a/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c +++ b/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c @@ -159,7 +159,7 @@ int MPIR_TSP_Ireduce_scatter_sched_intra_recexch(const void *sendbuf, void *recv /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); is_inplace = (sendbuf == MPI_IN_PLACE); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c b/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c index 5e4107de0b4..70aa2681896 100644 --- a/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c +++ b/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c @@ -32,7 +32,7 @@ int MPIR_TSP_Ireduce_scatter_block_sched_intra_recexch(const void *sendbuf, void /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); is_inplace = (sendbuf == MPI_IN_PLACE); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c b/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c index 2307607c8e1..34a24b05a5c 100644 --- a/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c +++ b/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c @@ -28,7 +28,7 @@ int MPIR_TSP_Iscan_sched_intra_recursive_doubling(const void *sendbuf, void *rec /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/iscatter/iscatter_tsp_tree.c b/src/mpi/coll/iscatter/iscatter_tsp_tree.c index 858f988050d..1c8aa9121e0 100644 --- a/src/mpi/coll/iscatter/iscatter_tsp_tree.c +++ b/src/mpi/coll/iscatter/iscatter_tsp_tree.c @@ -47,7 +47,7 @@ int MPIR_TSP_Iscatter_sched_intra_tree(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); if (rank == root && is_inplace) { diff --git a/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c b/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c index 1ede379982d..39fece505dc 100644 --- a/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c +++ b/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c @@ -32,7 +32,7 @@ int MPIR_TSP_Iscatterv_sched_allcomm_linear(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* If I'm the root, then scatter */ diff --git a/src/mpi/comm/contextid.c b/src/mpi/comm/contextid.c index d1d2833a435..d10b9833d6f 100644 --- a/src/mpi/comm/contextid.c +++ b/src/mpi/comm/contextid.c @@ -762,7 +762,7 @@ static int sched_cb_gcn_allocate_cid(MPIR_Comm * comm, int tag, void *state) * are not necessarily completed in the same order as they are issued, also on the * same communicator. To avoid deadlocks, we cannot add the elements to the * list bevfore the first iallreduce is completed. The "tag" is created for the - * scheduling - by calling MPIR_Sched_next_tag(comm_ptr, &tag) - and the same + * scheduling - by calling MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag) - and the same * for a idup operation on all processes. So we use it here. */ /* FIXME I'm not sure if there can be an overflows for this tag */ st->tag = (uint64_t) tag + MPIR_Process.attrs.tag_ub; @@ -945,7 +945,7 @@ int MPIR_Get_contextid_nonblock(MPIR_Comm * comm_ptr, MPIR_Comm * newcommp, MPIR MPIR_FUNC_ENTER; /* now create a schedule */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIR_Sched_create(&s, MPIR_SCHED_KIND_GENERALIZED); MPIR_ERR_CHECK(mpi_errno); @@ -986,7 +986,7 @@ int MPIR_Get_intercomm_contextid_nonblock(MPIR_Comm * comm_ptr, MPIR_Comm * newc } /* now create a schedule */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIR_Sched_create(&s, MPIR_SCHED_KIND_GENERALIZED); MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h b/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h index f0bc022a9db..65ad84d638b 100644 --- a/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h +++ b/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h @@ -90,7 +90,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf sizeof(struct fi_deferred_work), MPL_MEM_BUFFER); MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -110,7 +110,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf } i = i + j; - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -240,7 +240,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -261,7 +261,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer } i = i + j; - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -421,7 +421,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_rma(void *buffer MPIR_ERR_CHKANDJUMP1(*works == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "Triggered bcast deferred work alloc"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -567,7 +567,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_rma(void *buffer, i sizeof(struct fi_deferred_work), MPL_MEM_BUFFER); MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/ch4/netmod/ofi/ofi_events.c b/src/mpid/ch4/netmod/ofi/ofi_events.c index 84046946d11..b2ef6c8ebdf 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_events.c +++ b/src/mpid/ch4/netmod/ofi/ofi_events.c @@ -189,6 +189,7 @@ static int pipeline_recv_event(struct fi_cq_tagged_entry *wc, MPIR_Request * r, chunk_req->buf = host_buf; int ret = 0; if (!MPIDI_OFI_global.gpu_recv_queue && host_buf) { + /* FIXME: error handling */ ret = fi_trecv (MPIDI_OFI_global.ctx [MPIDI_OFI_REQUEST(rreq, pipeline_info.ctx_idx)].rx, diff --git a/src/mpid/ch4/netmod/ofi/ofi_recv.h b/src/mpid/ch4/netmod/ofi/ofi_recv.h index ffd66c98c35..a33018e3bcb 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_recv.h +++ b/src/mpid/ch4/netmod/ofi/ofi_recv.h @@ -271,6 +271,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf, chunk_req->parent = rreq; chunk_req->buf = host_buf; int ret = 0; + /* FIXME: handle error */ ret = fi_trecv(MPIDI_OFI_global.ctx[ctx_idx].rx, host_buf, MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ, diff --git a/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h b/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h index 148fcf472ce..16b245a0c6f 100644 --- a/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h +++ b/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h @@ -355,6 +355,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ibcast_impl(void *loc MPI_Aint type_size, nbytes, true_lb, true_extent; void *ori_local_buf = local_buf; MPI_Datatype ori_datatype = datatype; + int coll_group = MPIR_SUBGROUP_NONE; MPIR_CHKLMEM_DECL(1); /* Register the vertices */ @@ -425,7 +426,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ibcast_impl(void *loc MPIR_TSP_sched_malloc(sizeof(MPIDI_POSIX_per_call_ibcast_info_t), sched); MPIR_ERR_CHKANDJUMP(!data, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h b/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h index beedfab740d..cd6b2f239b2 100644 --- a/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h +++ b/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h @@ -364,6 +364,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ireduce_impl(void *se MPI_Aint num_chunks, chunk_count_floor, chunk_count_ceil; MPI_Aint true_extent, type_size, lb, extent; int offset = 0, is_contig; + int coll_group = MPIR_SUBGROUP_NONE; /* Register the vertices */ reserve_buf_type_id = MPIR_TSP_sched_new_type(sched, MPIDI_POSIX_NB_RG_rank0_hold_buf_issue, @@ -418,7 +419,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ireduce_impl(void *se data->seq_no = MPIDI_POSIX_COMM(comm_ptr, nb_reduce_seq_no); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/common/sched/mpidu_sched.c b/src/mpid/common/sched/mpidu_sched.c index e2fa385330f..0298b928c41 100644 --- a/src/mpid/common/sched/mpidu_sched.c +++ b/src/mpid/common/sched/mpidu_sched.c @@ -148,7 +148,7 @@ int MPIDU_Sched_are_pending(void) return (all_schedules.head != NULL); } -int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) +int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int coll_group, int *tag) { int mpi_errno = MPI_SUCCESS; /* TODO there should be an internal accessor/utility macro for getting the @@ -162,6 +162,10 @@ int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) MPIR_FUNC_ENTER; *tag = comm_ptr->next_sched_tag; + if (coll_group != MPIR_SUBGROUP_NONE) { + /* subgroup collectives use the same tag within a parent collective */ + goto fn_exit; + } ++comm_ptr->next_sched_tag; #if defined(HAVE_ERROR_CHECKING) @@ -191,11 +195,13 @@ int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) if (comm_ptr->next_sched_tag == tag_ub) { comm_ptr->next_sched_tag = MPIR_FIRST_NBC_TAG; } + fn_exit: + MPIR_FUNC_EXIT; + return mpi_errno; #if defined(HAVE_ERROR_CHECKING) fn_fail: + goto fn_exit; #endif - MPIR_FUNC_EXIT; - return mpi_errno; } void MPIDU_Sched_set_tag(struct MPIDU_Sched *s, int tag) diff --git a/src/mpid/common/sched/mpidu_sched.h b/src/mpid/common/sched/mpidu_sched.h index a61e64c6b69..90d43b75392 100644 --- a/src/mpid/common/sched/mpidu_sched.h +++ b/src/mpid/common/sched/mpidu_sched.h @@ -134,7 +134,7 @@ struct MPIDU_Sched { /* prototypes */ int MPIDU_Sched_progress(int vci, int *made_progress); int MPIDU_Sched_are_pending(void); -int MPIDU_Sched_next_tag(struct MPIR_Comm *comm_ptr, int *tag); +int MPIDU_Sched_next_tag(struct MPIR_Comm *comm_ptr, int coll_group, int *tag); void MPIDU_Sched_set_tag(MPIR_Sched_t s, int tag); int MPIDU_Sched_create(MPIR_Sched_t * sp, enum MPIR_Sched_kind kind); int MPIDU_Sched_clone(MPIR_Sched_t orig, MPIR_Sched_t * cloned);