Skip to content

use PMIx to manage CIDs #4674

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

use PMIx to manage CIDs #4674

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
24f7991
pmix/pmix3x: pass the returned info in the infocbfunc callback
ggouaillardet Jan 26, 2018
49b67cb
cid: add the new ompi cid framework
ggouaillardet Jan 19, 2018
60c6576
cid/pmix: add a new pmix component to the cid framework
ggouaillardet Jan 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ompi/communicator/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# All rights reserved.
# Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights
# reserved.
# Copyright (c) 2014 Research Organization for Information Science
# Copyright (c) 2014-2018 Research Organization for Information Science
# and Technology (RIST). All rights reserved.
# Copyright (c) 2016 IBM Corporation. All rights reserved.
# $COPYRIGHT$
Expand All @@ -31,6 +31,5 @@ headers += \
lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
communicator/comm_init.c \
communicator/comm.c \
communicator/comm_cid.c \
communicator/comm_request.c

34 changes: 18 additions & 16 deletions ompi/communicator/comm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* Copyright (c) 2012 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2012-2016 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2014-2017 Research Organization for Information Science
* Copyright (c) 2014-2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Mellanox Technologies. All rights reserved.
Expand Down Expand Up @@ -49,6 +49,8 @@

#include "ompi/attribute/attribute.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/cid/cid.h"
#include "ompi/mca/cid/base/base.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/request/request.h"

Expand Down Expand Up @@ -358,7 +360,7 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
Expand All @@ -368,7 +370,7 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
newcomp->c_contextid, comm->c_contextid );

/* Activate the communicator and init coll-component */
rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
Expand Down Expand Up @@ -593,15 +595,15 @@ int ompi_comm_split( ompi_communicator_t* comm, int color, int key,
}

/* set the rank to MPI_UNDEFINED. This prevents this process from interfering
* in ompi_comm_nextcid() and the collective module selection in ompi_comm_activate()
* in ompi_comm_nextcid() and the collective module selection in ompi_cid->activate()
* for a communicator that will be freed anyway.
*/
if ( MPI_UNDEFINED == color || (inter && my_rsize==0)) {
newcomp->c_local_group->grp_my_rank = MPI_UNDEFINED;
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
Expand All @@ -613,7 +615,7 @@ int ompi_comm_split( ompi_communicator_t* comm, int color, int key,


/* Activate the communicator and init coll-component */
rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->activate (&newcomp, comm, NULL, NULL, NULL, false, mode);

exit:
free ( results );
Expand Down Expand Up @@ -909,7 +911,7 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
break;
}
Expand All @@ -921,7 +923,7 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
}

/* Activate the communicator and init coll-component */
rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
break;
}
Expand Down Expand Up @@ -1004,7 +1006,7 @@ int ompi_comm_dup_with_info ( ompi_communicator_t * comm, opal_info_t *info, omp
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
return rc;
}
Expand All @@ -1020,7 +1022,7 @@ int ompi_comm_dup_with_info ( ompi_communicator_t * comm, opal_info_t *info, omp
}

/* activate communicator and init coll-module */
rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
rc = ompi_cid->activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
return rc;
}
Expand Down Expand Up @@ -1134,7 +1136,7 @@ static int ompi_comm_idup_getcid (ompi_comm_request_t *request)
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid_nb (context->newcomp, context->comm, NULL, NULL,
rc = ompi_cid->nextcid_nb (context->newcomp, context->comm, NULL, NULL,
NULL, false, mode, subreq);
if (OMPI_SUCCESS != rc) {
ompi_comm_request_return (request);
Expand Down Expand Up @@ -1164,7 +1166,7 @@ static int ompi_comm_idup_with_info_activate (ompi_comm_request_t *request)
context->newcomp->c_contextid, context->comm->c_contextid );

/* activate communicator and init coll-module */
rc = ompi_comm_activate_nb (&context->newcomp, context->comm, NULL, NULL, NULL, false, mode, subreq);
rc = ompi_cid->activate_nb (&context->newcomp, context->comm, NULL, NULL, NULL, false, mode, subreq);
if ( OMPI_SUCCESS != rc ) {
return rc;
}
Expand Down Expand Up @@ -1206,7 +1208,7 @@ int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int
}

/* Determine context id. It is identical to f_2_c_handle */
rc = ompi_comm_nextcid (newcomp, comm, NULL, &tag, NULL, false, mode);
rc = ompi_cid->nextcid (newcomp, comm, NULL, &tag, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
return rc;
}
Expand All @@ -1216,7 +1218,7 @@ int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int
newcomp->c_contextid, comm->c_contextid );

/* activate communicator and init coll-module */
rc = ompi_comm_activate (&newcomp, comm, NULL, &tag, NULL, false, mode);
rc = ompi_cid->activate (&newcomp, comm, NULL, &tag, NULL, false, mode);
if ( OMPI_SUCCESS != rc ) {
return rc;
}
Expand Down Expand Up @@ -1885,7 +1887,7 @@ int ompi_comm_enable(ompi_communicator_t *old_comm,
int ret = OMPI_SUCCESS;

/* Determine context id. It is identical to f_2_c_handle */
ret = ompi_comm_nextcid (new_comm, old_comm, NULL, NULL, NULL, false,
ret = ompi_cid->nextcid (new_comm, old_comm, NULL, NULL, NULL, false,
OMPI_COMM_CID_INTRA);
if (OMPI_SUCCESS != ret) {
/* something wrong happened while setting the communicator */
Expand All @@ -1909,7 +1911,7 @@ int ompi_comm_enable(ompi_communicator_t *old_comm,
goto complete_and_return;
}

ret = ompi_comm_activate (&new_comm, old_comm, NULL, NULL, NULL, false,
ret = ompi_cid->activate (&new_comm, old_comm, NULL, NULL, NULL, false,
OMPI_COMM_CID_INTRA);
if (OMPI_SUCCESS != ret) {
/* something wrong happened while setting the communicator */
Expand Down
6 changes: 4 additions & 2 deletions ompi/communicator/comm_init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* Copyright (c) 2011-2013 Inria. All rights reserved.
* Copyright (c) 2011-2013 Universite Bordeaux 1
* All rights reserved.
* Copyright (c) 2015-2017 Research Organization for Information Science
* Copyright (c) 2015-2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
Expand All @@ -45,6 +45,7 @@
#include "ompi/attribute/attribute.h"
#include "ompi/dpm/dpm.h"
#include "ompi/memchecker.h"
#include "ompi/mca/cid/cid.h"

/*
** Table for Fortran <-> C communicator handle conversion
Expand Down Expand Up @@ -335,7 +336,7 @@ int ompi_comm_finalize(void)
* the reference count by one more than other communicators, on order to
* allow for deallocation with the parent communicator. Note, that
* this only occurs if the cid of the local_comm is lower than of its
* parent communicator. Read the comment in comm_activate for
* parent communicator. Read the comment in ompi_cid_base_comm_activate() for
* a full explanation.
*/
if ( ompi_debug_show_handle_leaks && !(OMPI_COMM_IS_FREED(comm)) ){
Expand Down Expand Up @@ -456,6 +457,7 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
comm->c_contextid)) {
opal_pointer_array_set_item ( &ompi_mpi_communicators,
comm->c_contextid, NULL);
ompi_cid->release(comm->c_contextid);
}

/* reset the ompi_comm_f_to_c_table entry */
Expand Down
32 changes: 25 additions & 7 deletions ompi/communicator/comm_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
/*
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
* reseved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
Expand All @@ -29,7 +29,8 @@ bool ompi_comm_request_initialized = false;
typedef struct ompi_comm_request_item_t {
opal_list_item_t super;
ompi_comm_request_callback_fn_t callback;
ompi_request_t *subreqs[OMPI_COMM_REQUEST_MAX_SUBREQ];
ompi_request_t *static_subreqs[OMPI_COMM_REQUEST_MAX_SUBREQ];
ompi_request_t **subreqs;
int subreq_count;
} ompi_comm_request_item_t;
OBJ_CLASS_DECLARATION(ompi_comm_request_item_t);
Expand Down Expand Up @@ -74,15 +75,20 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r
ompi_comm_request_item_t *request_item;
int i;

if (subreq_count > OMPI_COMM_REQUEST_MAX_SUBREQ) {
return OMPI_ERR_BAD_PARAM;
}

request_item = OBJ_NEW(ompi_comm_request_item_t);
if (NULL == request_item) {
return OMPI_ERR_OUT_OF_RESOURCE;
}

if (subreq_count > OMPI_COMM_REQUEST_MAX_SUBREQ) {
ompi_request_t ** reqs = (ompi_request_t **)malloc(subreq_count * sizeof(ompi_request_t *));
if (NULL == reqs) {
OBJ_RELEASE(request_item);
return OMPI_ERR_OUT_OF_RESOURCE;
}
request_item->subreqs = reqs;
}

request_item->callback = callback;

for (i = 0 ; i < subreq_count ; ++i) {
Expand Down Expand Up @@ -241,7 +247,19 @@ OBJ_CLASS_INSTANCE(ompi_comm_request_t, ompi_request_t,
ompi_comm_request_construct,
ompi_comm_request_destruct);

OBJ_CLASS_INSTANCE(ompi_comm_request_item_t, opal_list_item_t, NULL, NULL);
static void ompi_comm_request_item_construct (ompi_comm_request_item_t *request) {
request->subreqs = request->static_subreqs;
}

static void ompi_comm_request_item_destruct (ompi_comm_request_item_t *request) {
if (request->static_subreqs != request->subreqs) {
free(request->subreqs);
}
}

OBJ_CLASS_INSTANCE(ompi_comm_request_item_t, opal_list_item_t,
ompi_comm_request_item_construct,
ompi_comm_request_item_destruct);

ompi_comm_request_t *ompi_comm_request_get (void)
{
Expand Down
75 changes: 2 additions & 73 deletions ompi/communicator/communicator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
* $COPYRIGHT$
Expand Down Expand Up @@ -92,6 +92,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
#define OMPI_COMM_ALLGATHER_TAG -31078
#define OMPI_COMM_BARRIER_TAG -31079
#define OMPI_COMM_ALLREDUCE_TAG -31080
#define OMPI_COMM_LOCAL_TAG -31081

#define OMPI_COMM_ASSERT_NO_ANY_TAG 0x00000001
#define OMPI_COMM_ASSERT_NO_ANY_SOURCE 0x00000002
Expand Down Expand Up @@ -537,50 +538,6 @@ OMPI_DECLSPEC int ompi_comm_free (ompi_communicator_t **comm);
ompi_communicator_t* ompi_comm_allocate (int local_group_size,
int remote_group_size);

/**
* allocate new communicator ID
* @param newcomm: pointer to the new communicator
* @param oldcomm: original comm
* @param bridgecomm: bridge comm for intercomm_create
* @param mode: combination of input
* OMPI_COMM_CID_INTRA: intra-comm
* OMPI_COMM_CID_INTER: inter-comm
* OMPI_COMM_CID_GROUP: only decide CID within the ompi_group_t
* associated with the communicator. arg0
* must point to an int which will be used
* as the pml tag for communication.
* OMPI_COMM_CID_INTRA_BRIDGE: 2 intracomms connected by
* a bridge comm. arg0 and arg1 must point
* to integers representing the local and
* remote leader ranks. the remote leader rank
* is a rank in the bridgecomm.
* OMPI_COMM_CID_INTRA_PMIX: 2 intracomms, leaders talk
* through PMIx. arg0 must point to an integer
* representing the local leader rank. arg1
* must point to a string representing the
* port of the remote leader.
* @param send_first: to avoid a potential deadlock for
* the OOB version.
* This routine has to be thread safe in the final version.
*/
OMPI_DECLSPEC int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
bool send_first, int mode);

/**
* allocate new communicator ID (non-blocking)
* @param newcomm: pointer to the new communicator
* @param oldcomm: original comm
* @param bridgecomm: bridge comm for intercomm_create
* @param mode: combination of input
* OMPI_COMM_CID_INTRA: intra-comm
* OMPI_COMM_CID_INTER: inter-comm
* This routine has to be thread safe in the final version.
*/
OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
bool send_first, int mode, ompi_request_t **req);

/**
* shut down the communicator infrastructure.
*/
Expand Down Expand Up @@ -672,26 +629,6 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
int high );


OMPI_DECLSPEC int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
ompi_communicator_t *bridgecomm, const void *arg0,
const void *arg1, bool send_first, int mode);

/**
* Non-blocking variant of comm_activate.
*
* @param[inout] newcomm New communicator
* @param[in] comm Parent communicator
* @param[in] bridgecomm Bridge communicator (used for PMIX and bridge modes)
* @param[in] arg0 Mode argument 0
* @param[in] arg1 Mode argument 1
* @param[in] send_first Send first from this process (PMIX mode only)
* @param[in] mode Collective mode
* @param[out] req New request object to track this operation
*/
OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
ompi_communicator_t *bridgecomm, const void *arg0,
const void *arg1, bool send_first, int mode, ompi_request_t **req);

/**
* a simple function to dump the structure
*/
Expand All @@ -704,14 +641,6 @@ int ompi_comm_set_name (ompi_communicator_t *comm, const char *name );
extern int ompi_comm_num_dyncomm;


/* check whether any of the processes has requested support for
MPI_THREAD_MULTIPLE. Note, that this produces global
information across MPI_COMM_WORLD, in contrary to the local
flag ompi_mpi_thread_provided
*/
OMPI_DECLSPEC int ompi_comm_cid_init ( void );


void ompi_comm_assert_subscribe (ompi_communicator_t *comm, int32_t assert_flag);

END_C_DECLS
Expand Down
Loading