From e05a4ff44ecd24446b6aebe035cc563995579cbc Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 14 Oct 2015 15:43:13 -0600 Subject: [PATCH 1/5] opal/mutex: add static mutex initializers Signed-off-by: Nathan Hjelm (cherry picked from commit open-mpi/ompi@039c7dbcd6bd9d2915097cdb2dcb747bc382c356) --- opal/class/opal_object.h | 15 ++++++++++-- opal/threads/mutex_unix.h | 51 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 8045904990..e8ed5474f1 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -165,9 +165,20 @@ struct opal_class_t { * @param NAME Name of the class to initialize */ #if OPAL_ENABLE_DEBUG -#define OPAL_OBJ_STATIC_INIT(BASE_CLASS) { OPAL_OBJ_MAGIC_ID, OBJ_CLASS(BASE_CLASS), 1, __FILE__, __LINE__ } +#define OPAL_OBJ_STATIC_INIT(BASE_CLASS) \ + { \ + .obj_magic_id = OPAL_OBJ_MAGIC_ID, \ + .obj_class = OBJ_CLASS(BASE_CLASS), \ + .obj_reference_count = 1, \ + .cls_init_file_name = __FILE__, \ + .cls_init_lineno = __LINE__, \ + } #else -#define OPAL_OBJ_STATIC_INIT(BASE_CLASS) { OBJ_CLASS(BASE_CLASS), 1 } +#define OPAL_OBJ_STATIC_INIT(BASE_CLASS) \ + { \ + .obj_class = OBJ_CLASS(BASE_CLASS), \ + .obj_reference_count = 1, \ + } #endif /** diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index d159285d8e..a2de825928 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -9,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -63,6 +64,54 @@ struct opal_mutex_t { OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); +#if defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP) +#define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP +#elif defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER) +#define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER +#endif + +#if OPAL_ENABLE_DEBUG +#define OPAL_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ + .m_lock_debug = 0, \ + .m_lock_file = NULL, \ + .m_lock_line = 0, \ + .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_UNLOCKED } }, \ + } +#else +#define OPAL_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ + .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_UNLOCKED } }, \ + } +#endif + +#if defined(OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER) + +#if OPAL_ENABLE_DEBUG +#define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ + .m_lock_debug = 0, \ + .m_lock_file = NULL, \ + .m_lock_line = 0, \ + .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_UNLOCKED } }, \ + } +#else +#define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ + .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_UNLOCKED } }, \ + } +#endif + +#endif + /************************************************************************ * * mutex operations (non-atomic versions) From 73e49f8f0e7f7ad725b5fd39bb1cc918718ebe83 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 14 Oct 2015 15:43:32 -0600 Subject: [PATCH 2/5] mpit: use opal static mutex initializer Signed-off-by: Nathan Hjelm (cherry picked from commit open-mpi/ompi@7f7ff8d851dca7463d3c8ea6e1525096c0073c40) --- ompi/mpi/tool/init_thread.c | 10 ---------- ompi/mpi/tool/mpit_common.c | 11 +++-------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/ompi/mpi/tool/init_thread.c b/ompi/mpi/tool/init_thread.c index 478d3e9b16..8f0fb6b3c6 100644 --- a/ompi/mpi/tool/init_thread.c +++ b/ompi/mpi/tool/init_thread.c @@ -32,18 +32,8 @@ extern volatile int32_t initted; int MPI_T_init_thread (int required, int *provided) { - static volatile int32_t first_init = 1; int rc = MPI_SUCCESS; - if (opal_atomic_cmpset (&first_init, 1, 0) == 1) { - OBJ_CONSTRUCT(&mpit_big_lock, opal_mutex_t); - initted = 1; - } - - while (!initted) { - usleep (10); - } - mpit_lock (); do { diff --git a/ompi/mpi/tool/mpit_common.c b/ompi/mpi/tool/mpit_common.c index 98920aeb21..7756889b8f 100644 --- a/ompi/mpi/tool/mpit_common.c +++ b/ompi/mpi/tool/mpit_common.c @@ -13,23 +13,18 @@ #include "ompi/mpi/tool/mpit-internal.h" -opal_mutex_t mpit_big_lock = {{0}}; +opal_mutex_t mpit_big_lock = OPAL_MUTEX_STATIC_INIT; volatile uint32_t mpit_init_count = 0; -volatile int32_t initted = 0; void mpit_lock (void) { - if (initted) { - opal_mutex_lock (&mpit_big_lock); - } + opal_mutex_lock (&mpit_big_lock); } void mpit_unlock (void) { - if (initted) { - opal_mutex_unlock (&mpit_big_lock); - } + opal_mutex_unlock (&mpit_big_lock); } int ompit_var_type_to_datatype (mca_base_var_type_t type, MPI_Datatype *datatype) From 8850877896c524f2cd23dda5f203e5ca48197111 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sun, 11 Oct 2015 07:31:47 -0500 Subject: [PATCH 3/5] init/finalize: updates for MPI-3.1 - If MPI_INITLIZED is invoked and MPI is only partially initialized, wait until MPI is fully initialized before returning. - If MPI_FINALIZED is invoked and MPI is only partially finalized, wait until MPI is fully finalized before returning. (cherry picked from commit open-mpi/ompi@f5ad90c92053e4a60ab7c203b40577026f567db9) --- ompi/mpi/c/finalized.c | 17 +++++- ompi/mpi/c/init.c | 21 +------ ompi/mpi/c/init_thread.c | 20 +------ ompi/mpi/c/initialized.c | 17 +++++- ompi/runtime/help-mpi-runtime.txt | 18 ++++++ ompi/runtime/mpiruntime.h | 21 ++++--- ompi/runtime/ompi_mpi_finalize.c | 97 ++++++++++++++++++------------- ompi/runtime/ompi_mpi_init.c | 36 +++++++++--- 8 files changed, 148 insertions(+), 99 deletions(-) diff --git a/ompi/mpi/c/finalized.c b/ompi/mpi/c/finalized.c index c6b3f96199..c1de8eeb0c 100644 --- a/ompi/mpi/c/finalized.c +++ b/ompi/mpi/c/finalized.c @@ -9,6 +9,9 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,6 +41,14 @@ int MPI_Finalized(int *flag) { MPI_Comm null = NULL; + /* We must obtain the lock to guarnatee consistent values of + ompi_mpi_initialized and ompi_mpi_finalized. Note, too, that + this lock is held for the bulk of the duration of + ompi_mpi_init() and ompi_mpi_finalize(), so when we get the + lock, we are guaranteed that some other thread is not part way + through initialization or finalization. */ + opal_mutex_lock(&ompi_mpi_bootstrap_mutex); + if (MPI_PARAM_CHECK) { if (NULL == flag) { @@ -46,17 +57,19 @@ int MPI_Finalized(int *flag) MPI_Finalize) or not */ if (ompi_mpi_initialized && !ompi_mpi_finalized) { + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, FUNC_NAME); } else { + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG, FUNC_NAME); } } } - /* Pretty simple */ - *flag = ompi_mpi_finalized; + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + return MPI_SUCCESS; } diff --git a/ompi/mpi/c/init.c b/ompi/mpi/c/init.c index ac04063f9d..5154557bfa 100644 --- a/ompi/mpi/c/init.c +++ b/ompi/mpi/c/init.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * $COPYRIGHT$ * @@ -46,25 +46,6 @@ int MPI_Init(int *argc, char ***argv) char *env; int required = MPI_THREAD_SINGLE; - /* Ensure that we were not already initialized or finalized */ - - if (ompi_mpi_finalized) { - if (0 == ompi_comm_rank(MPI_COMM_WORLD)) { - opal_show_help("help-mpi-api.txt", - "mpi-function-after-finalize", true, FUNC_NAME); - } - return ompi_errhandler_invoke(NULL, NULL, - OMPI_ERRHANDLER_TYPE_COMM, - MPI_ERR_OTHER, FUNC_NAME); - } else if (ompi_mpi_initialized) { - if (0 == ompi_comm_rank(MPI_COMM_WORLD)) { - opal_show_help("help-mpi-api.txt", "mpi-initialize-twice", - true, FUNC_NAME); - } - return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, - FUNC_NAME); - } - /* check for environment overrides for required thread level. If there is, check to see that it is a valid/supported thread level. If not, default to MPI_THREAD_MULTIPLE. */ diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c index f70ef43b3b..e3857ae446 100644 --- a/ompi/mpi/c/init_thread.c +++ b/ompi/mpi/c/init_thread.c @@ -10,6 +10,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -59,23 +62,6 @@ int MPI_Init_thread(int *argc, char ***argv, int required, *provided = MPI_THREAD_SINGLE; #endif - /* Ensure that we were not already initialized or finalized */ - - if (ompi_mpi_finalized) { - if (0 == ompi_comm_rank(MPI_COMM_WORLD)) { - opal_show_help("help-mpi-api.txt", "mpi-function-after-finalize", - true, FUNC_NAME); - } - return ompi_errhandler_invoke(NULL, NULL, OMPI_ERRHANDLER_TYPE_COMM, - MPI_ERR_OTHER, FUNC_NAME); - } else if (ompi_mpi_initialized) { - if (0 == ompi_comm_rank(MPI_COMM_WORLD)) { - opal_show_help("help-mpi-api.txt", "mpi-initialize-twice", - true, FUNC_NAME); - } - return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, FUNC_NAME); - } - /* Call the back-end initialization function (we need to put as little in this function as possible so that if it's profiled, we don't lose anything) */ diff --git a/ompi/mpi/c/initialized.c b/ompi/mpi/c/initialized.c index 0e4397a11b..a218518038 100644 --- a/ompi/mpi/c/initialized.c +++ b/ompi/mpi/c/initialized.c @@ -9,6 +9,9 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,6 +41,14 @@ int MPI_Initialized(int *flag) { MPI_Comm null = NULL; + /* We must obtain the lock to guarnatee consistent values of + ompi_mpi_initialized and ompi_mpi_finalized. Note, too, that + this lock is held for the bulk of the duration of + ompi_mpi_init() and ompi_mpi_finalize(), so when we get the + lock, we are guaranteed that some other thread is not part way + through initialization or finalization. */ + opal_mutex_lock(&ompi_mpi_bootstrap_mutex); + if (MPI_PARAM_CHECK) { if (NULL == flag) { @@ -46,17 +57,19 @@ int MPI_Initialized(int *flag) MPI_Finalize) or not */ if (ompi_mpi_initialized && !ompi_mpi_finalized) { + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, FUNC_NAME); } else { + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG, FUNC_NAME); } } } - /* Pretty simple */ - *flag = ompi_mpi_initialized; + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + return MPI_SUCCESS; } diff --git a/ompi/runtime/help-mpi-runtime.txt b/ompi/runtime/help-mpi-runtime.txt index 0db4654d97..00fa9a042d 100644 --- a/ompi/runtime/help-mpi-runtime.txt +++ b/ompi/runtime/help-mpi-runtime.txt @@ -50,6 +50,24 @@ You may wish to try to narrow down the problem; WARNING: The MCA parameter mpi_param_check has been set to true, but parameter checking has been compiled out of Open MPI. The mpi_param_check value has therefore been ignored. +# +[mpi_init: invoked multiple times] +Open MPI has detected that this process has attempted to initialize +MPI (via MPI_INIT or MPI_INIT_THREAD) more than once. This is +erroneous. +# +[mpi_init: already finalized] +Open MPI has detected that this process has attempted to initialize +MPI (via MPI_INIT or MPI_INIT_THREAD) after MPI_FINALIZE has been +called. This is erroneous. +# +[mpi_finalize: not initialized] +The function MPI_FINALIZE was invoked before MPI was initialized in a +process on host %s, PID %d. + +This indicates an erroneous MPI program; MPI must be initialized +before it can be finalized. +# [mpi_finalize:invoked_multiple_times] The function MPI_FINALIZE was invoked multiple times in a single process on host %s, PID %d. diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h index af050f66a3..7efde6fd25 100644 --- a/ompi/runtime/mpiruntime.h +++ b/ompi/runtime/mpiruntime.h @@ -35,6 +35,7 @@ #include "opal/class/opal_list.h" #include "opal/class/opal_hash_table.h" +#include "opal/threads/mutex.h" BEGIN_C_DECLS @@ -47,16 +48,18 @@ struct ompi_predefined_datatype_t; /* Global variables and symbols for the MPI layer */ -/** Did mpi start to initialize? */ -OMPI_DECLSPEC extern bool ompi_mpi_init_started; -/** Is mpi initialized? */ -OMPI_DECLSPEC extern bool ompi_mpi_initialized; -/** Has mpi been finalized? */ -OMPI_DECLSPEC extern bool ompi_mpi_finalized; +/** Mutex to protect all the _init and _finalize variables */ +OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex; +/** Did MPI start to initialize? */ +OMPI_DECLSPEC extern volatile bool ompi_mpi_init_started; /** Has the RTE been initialized? */ -OMPI_DECLSPEC extern bool ompi_rte_initialized; -/** Did mpi start to finalize? */ -OMPI_DECLSPEC extern int32_t ompi_mpi_finalize_started; +OMPI_DECLSPEC extern volatile bool ompi_rte_initialized; +/** Is MPI fully initialized? */ +OMPI_DECLSPEC extern volatile bool ompi_mpi_initialized; +/** Did MPI start to finalize? */ +OMPI_DECLSPEC extern volatile bool ompi_mpi_finalize_started; +/** Has MPI been fully finalized? */ +OMPI_DECLSPEC extern volatile bool ompi_mpi_finalized; /** Do we have multiple threads? */ OMPI_DECLSPEC extern bool ompi_mpi_thread_multiple; diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c index 9c667130fb..575db3e873 100644 --- a/ompi/runtime/ompi_mpi_finalize.c +++ b/ompi/runtime/ompi_mpi_finalize.c @@ -92,32 +92,44 @@ extern bool ompi_enable_timing_ext; int ompi_mpi_finalize(void) { - int ret; + int ret = MPI_SUCCESS; opal_list_item_t *item; ompi_proc_t** procs; size_t nprocs; OPAL_TIMING_DECLARE(tm); OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY); - /* Be a bit social if an erroneous program calls MPI_FINALIZE in two different threads, otherwise we may deadlock in ompi_comm_free() (or run into other nasty lions, tigers, or - bears) */ - - if (! opal_atomic_cmpset_32(&ompi_mpi_finalize_started, 0, 1)) { - /* Note that if we're already finalized, we cannot raise an - MPI exception. The best that we can do is write something - to stderr. */ + bears). + + This lock is held for the duration of ompi_mpi_init() and + ompi_mpi_finalize(). Hence, if we get it, then no other thread + is inside the critical section (and we don't have to check the + *_started bool variables). */ + opal_mutex_lock(&ompi_mpi_bootstrap_mutex); + if (!ompi_mpi_initialized || ompi_mpi_finalized) { + /* Note that if we're not initialized or already finalized, we + cannot raise an MPI exception. The best that we can do is + write something to stderr. */ char hostname[MAXHOSTNAMELEN]; pid_t pid = getpid(); gethostname(hostname, sizeof(hostname)); - opal_show_help("help-mpi-runtime.txt", - "mpi_finalize:invoked_multiple_times", - true, hostname, pid); + if (ompi_mpi_initialized) { + opal_show_help("help-mpi-runtime.txt", + "mpi_finalize: not initialized", + true, hostname, pid); + } else if (ompi_mpi_finalized) { + opal_show_help("help-mpi-runtime.txt", + "mpi_finalize:invoked_multiple_times", + true, hostname, pid); + } + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return MPI_ERR_OTHER; } + ompi_mpi_finalize_started = true; ompi_mpiext_fini(); @@ -267,21 +279,21 @@ int ompi_mpi_finalize(void) /* free file resources */ if (OMPI_SUCCESS != (ret = ompi_file_finalize())) { - return ret; + goto done; } /* free window resources */ if (OMPI_SUCCESS != (ret = ompi_win_finalize())) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = ompi_osc_base_finalize())) { - return ret; + goto done; } /* free communicator resources. this MUST come before finalizing the PML * as this will call into the pml */ if (OMPI_SUCCESS != (ret = ompi_comm_finalize())) { - return ret; + goto done; } nprocs = 0; @@ -291,16 +303,16 @@ int ompi_mpi_finalize(void) /* free pml resource */ if(OMPI_SUCCESS != (ret = mca_pml_base_finalize())) { - return ret; + goto done; } /* free requests */ if (OMPI_SUCCESS != (ret = ompi_request_finalize())) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = ompi_message_finalize())) { - return ret; + goto done; } /* If requested, print out a list of memory allocated by ALLOC_MEM @@ -313,7 +325,7 @@ int ompi_mpi_finalize(void) shut down MCA types having to do with communications */ if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_pml_base_framework) ) ) { OMPI_ERROR_LOG(ret); - return ret; + goto done; } /* shut down buffered send code */ @@ -325,7 +337,7 @@ int ompi_mpi_finalize(void) */ if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_crcp_base_framework) ) ) { OMPI_ERROR_LOG(ret); - return ret; + goto done; } #endif @@ -333,49 +345,49 @@ int ompi_mpi_finalize(void) /* free attr resources */ if (OMPI_SUCCESS != (ret = ompi_attr_finalize())) { - return ret; + goto done; } /* free group resources */ if (OMPI_SUCCESS != (ret = ompi_group_finalize())) { - return ret; + goto done; } /* finalize the DPM subsystem */ if ( OMPI_SUCCESS != (ret = ompi_dpm_finalize())) { - return ret; + goto done; } /* free internal error resources */ if (OMPI_SUCCESS != (ret = ompi_errcode_intern_finalize())) { - return ret; + goto done; } /* free error code resources */ if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_finalize())) { - return ret; + goto done; } /* free errhandler resources */ if (OMPI_SUCCESS != (ret = ompi_errhandler_finalize())) { - return ret; + goto done; } /* Free all other resources */ /* free op resources */ if (OMPI_SUCCESS != (ret = ompi_op_finalize())) { - return ret; + goto done; } /* free ddt resources */ if (OMPI_SUCCESS != (ret = ompi_datatype_finalize())) { - return ret; + goto done; } /* free info resources */ if (OMPI_SUCCESS != (ret = ompi_info_finalize())) { - return ret; + goto done; } /* Close down MCA modules */ @@ -387,32 +399,32 @@ int ompi_mpi_finalize(void) ompi_io_base_framework.framework_refcnt = 1; if (OMPI_SUCCESS != mca_base_framework_close(&ompi_io_base_framework)) { - return ret; + goto done; } } (void) mca_base_framework_close(&ompi_topo_base_framework); if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_osc_base_framework))) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_coll_base_framework))) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_bml_base_framework))) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_mpool_base_framework))) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_rcache_base_framework))) { - return ret; + goto done; } if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_allocator_base_framework))) { - return ret; + goto done; } /* free proc resources */ if ( OMPI_SUCCESS != (ret = ompi_proc_finalize())) { - return ret; + goto done; } if (NULL != ompi_mpi_main_thread) { @@ -423,21 +435,24 @@ int ompi_mpi_finalize(void) /* Leave the RTE */ if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) { - return ret; + goto done; } ompi_rte_initialized = false; /* now close the rte framework */ if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_rte_base_framework) ) ) { OMPI_ERROR_LOG(ret); - return ret; + goto done; } if (OPAL_SUCCESS != (ret = opal_finalize_util())) { - return ret; + goto done; } /* All done */ - return MPI_SUCCESS; + done: + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + + return ret; } diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 8d8c571c87..7bff391e47 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -124,11 +124,12 @@ const char ompi_version_string[] = OMPI_IDENT_STRING; * Global variables and symbols for the MPI layer */ -bool ompi_mpi_init_started = false; -bool ompi_mpi_initialized = false; -bool ompi_mpi_finalized = false; -bool ompi_rte_initialized = false; -int32_t ompi_mpi_finalize_started = false; +opal_mutex_t ompi_mpi_bootstrap_mutex = OPAL_MUTEX_STATIC_INIT; +volatile bool ompi_mpi_init_started = false; +volatile bool ompi_mpi_initialized = false; +volatile bool ompi_mpi_finalize_started = false; +volatile bool ompi_mpi_finalized = false; +volatile bool ompi_rte_initialized = false; bool ompi_mpi_thread_multiple = false; int ompi_mpi_thread_requested = MPI_THREAD_SINGLE; @@ -384,9 +385,26 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) * for the modex in order to work in heterogeneous environments. */ uint8_t threadlevel_bf; - /* Indicate that we have *started* MPI_INIT*. MPI_FINALIZE has - something sorta similar in a static local variable in - ompi_mpi_finalize(). */ + /* Ensure that we were not already initialized or finalized. + + This lock is held for the duration of ompi_mpi_init() and + ompi_mpi_finalize(). Hence, if we get it, then no other thread + is inside the critical section (and we don't have to check the + *_started bool variables). */ + opal_mutex_lock(&ompi_mpi_bootstrap_mutex); + if (ompi_mpi_finalized) { + opal_show_help("help-mpi-runtime.txt", + "mpi_init: already finalized", true); + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + return MPI_ERR_OTHER; + } else if (ompi_mpi_initialized) { + opal_show_help("help-mpi-runtime.txt", + "mpi_init: invoked multiple times", true); + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + return MPI_ERR_OTHER; + } + + /* Indicate that we have *started* MPI_INIT* */ ompi_mpi_init_started = true; /* Setup enough to check get/set MCA params */ @@ -894,6 +912,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) "mpi_init:startup:internal-failure", true, "MPI_INIT", "MPI_INIT", error, err_msg, ret); } + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return ret; } @@ -923,5 +942,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) OPAL_TIMING_REPORT(ompi_enable_timing_ext, &tm); OPAL_TIMING_RELEASE(&tm); + opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); return MPI_SUCCESS; } From a46e6d61127ccc3964fe479b701adcd5b91bd5d9 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Tue, 13 Oct 2015 10:01:40 -0400 Subject: [PATCH 4/5] man: update man pages for Init*/Finalize* Update language surrounding initialization and finalization in MPI_Init[_thread], MPI_Initialized, MPI_Finalize, and MPI_Finalized. (cherry picked from commit open-mpi/ompi@338257a2f4b04671b205d13142f03bd305c48f7d) --- ompi/mpi/man/man3/MPI_Finalize.3in | 9 ++++++++- ompi/mpi/man/man3/MPI_Finalized.3in | 18 ++++++++++++++---- ompi/mpi/man/man3/MPI_Init.3in | 12 ++++++------ ompi/mpi/man/man3/MPI_Init_thread.3in | 9 +++++---- ompi/mpi/man/man3/MPI_Initialized.3in | 17 +++++++++++------ 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_Finalize.3in b/ompi/mpi/man/man3/MPI_Finalize.3in index a4ffcc689b..9c6d7cc27c 100644 --- a/ompi/mpi/man/man3/MPI_Finalize.3in +++ b/ompi/mpi/man/man3/MPI_Finalize.3in @@ -1,5 +1,5 @@ .\" -*- nroff -*- -.\" Copyright 2010 Cisco Systems, Inc. All rights reserved. +.\" Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. .\" Copyright 2006-2008 Sun Microsystems, Inc. .\" Copyright (c) 1996 Thinking Machines Corporation .\" $COPYRIGHT$ @@ -55,3 +55,10 @@ Before the error value is returned, the current MPI error handler is called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error. +.SH SEE ALSO +.ft R +.nf +MPI_Init +MPI_Init_thread +MPI_Initialized +MPI_Finalized diff --git a/ompi/mpi/man/man3/MPI_Finalized.3in b/ompi/mpi/man/man3/MPI_Finalized.3in index dae99f003b..db1b893a34 100644 --- a/ompi/mpi/man/man3/MPI_Finalized.3in +++ b/ompi/mpi/man/man3/MPI_Finalized.3in @@ -1,11 +1,11 @@ .\" -*- nroff -*- -.\" Copyright 2010 Cisco Systems, Inc. All rights reserved. +.\" Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. .\" Copyright 2006-2008 Sun Microsystems, Inc. .\" Copyright (c) 1996 Thinking Machines Corporation .\" $COPYRIGHT$ .TH MPI_Finalized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" .SH NAME -\fBMPI_Finalized \fP \- Checks whether MPI_Finalize has completed. +\fBMPI_Finalized \fP \- Checks whether MPI has been finalized .SH SYNTAX .ft R @@ -33,7 +33,7 @@ bool MPI::Is_finalized() .ft R .TP 1i flag -True if MPI was finalized (logical). +True if MPI was finalized, and false otherwise (logical). .ft R .TP 1i IERROR @@ -41,7 +41,10 @@ Fortran only: Error status (integer). .SH DESCRIPTION .ft R -This routine returns "true" if MPI_Finalize has completed. You can call MPI_Finalized before MPI_Init and after MPI_Finalize. +This routine may be used to determine whether MPI has been finalized. +It is one of a small number of routines that may be called before MPI +is initialized and after MPI has been finalized (MPI_Initialized is +another). .SH ERRORS Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object. @@ -49,3 +52,10 @@ Almost all MPI routines return an error value; C routines as the value of the fu Before the error value is returned, the current MPI error handler is called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error. +.SH SEE ALSO +.ft R +.nf +MPI_Init +MPI_Init_thread +MPI_Initialized +MPI_Finalize diff --git a/ompi/mpi/man/man3/MPI_Init.3in b/ompi/mpi/man/man3/MPI_Init.3in index 603170260e..d2cc5abc72 100644 --- a/ompi/mpi/man/man3/MPI_Init.3in +++ b/ompi/mpi/man/man3/MPI_Init.3in @@ -1,5 +1,5 @@ .\" -*- nroff -*- -.\" Copyright 2010 Cisco Systems, Inc. All rights reserved. +.\" Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. .\" Copyright 2006-2008 Sun Microsystems, Inc. .\" Copyright (c) 1996 Thinking Machines Corporation .\" $COPYRIGHT$ @@ -48,10 +48,10 @@ Fortran only: Error status (integer). .SH DESCRIPTION .ft R -This routine, or MPI_Init_thread, must be called before any other MPI -routine (apart from MPI_Initialized) is called. MPI can be initialized -at most once; subsequent calls to MPI_Init or MPI_Init_thread are -erroneous. +This routine, or MPI_Init_thread, must be called before most other MPI +routines are called. There are a small number of exceptions, such as +MPI_Initialized and MPI_Finalized. MPI can be initialized at most +once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous. .sp All MPI programs must contain a call to MPI_Init or MPI_Init_thread. Open MPI accepts the C/C++ \fIargc\fP and \fIargv\fP @@ -102,4 +102,4 @@ See the MPI man page for a full list of MPI error codes. MPI_Init_thread MPI_Initialized MPI_Finalize - +MPI_Finalized diff --git a/ompi/mpi/man/man3/MPI_Init_thread.3in b/ompi/mpi/man/man3/MPI_Init_thread.3in index 2362235804..4383eb0fc2 100644 --- a/ompi/mpi/man/man3/MPI_Init_thread.3in +++ b/ompi/mpi/man/man3/MPI_Init_thread.3in @@ -1,7 +1,7 @@ .\" -*- nroff -*- .\" Copyright 2006-2008 Sun Microsystems, Inc. .\" Copyright (c) 1996 Thinking Machines Corporation -.\" Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +.\" Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. .\" $COPYRIGHT$ .TH MPI_Init_thread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" . @@ -57,8 +57,9 @@ Fortran only: Error status (integer). . .SH DESCRIPTION .ft R -This routine, or MPI_Init, must be called before any other MPI routine -(apart from MPI_Initialized) is called. MPI can be initialized at most +This routine, or MPI_Init, must be called before most other MPI +routines are called. There are a small number of exceptions, such as +MPI_Initialized and MPI_Finalized. MPI can be initialized at most once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous. .sp MPI_Init_thread, as compared to MPI_Init, has a provision to request a @@ -193,4 +194,4 @@ guarantee that an MPI program can continue past an error. MPI_Init MPI_Initialized MPI_Finalize - +MPI_Finalized diff --git a/ompi/mpi/man/man3/MPI_Initialized.3in b/ompi/mpi/man/man3/MPI_Initialized.3in index 215967912b..d92cf39b03 100644 --- a/ompi/mpi/man/man3/MPI_Initialized.3in +++ b/ompi/mpi/man/man3/MPI_Initialized.3in @@ -1,11 +1,11 @@ .\" -*- nroff -*- -.\" Copyright 2010 Cisco Systems, Inc. All rights reserved. +.\" Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. .\" Copyright 2006-2008 Sun Microsystems, Inc. .\" Copyright (c) 1996 Thinking Machines Corporation .\" $COPYRIGHT$ .TH MPI_Initialized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" .SH NAME -\fBMPI_Initialized\fP \- Indicates whether MPI_Init has been called. +\fBMPI_Initialized\fP \- Checks whether MPI has been initialized .SH SYNTAX .ft R @@ -33,7 +33,7 @@ bool Is_initialized() .ft R .TP 1i flag -Flag is true if MPI_Init has been called and false otherwise. +True if MPI has been initialized, and false otherwise (logical). .ft R .TP 1i IERROR @@ -41,7 +41,10 @@ Fortran only: Error status (integer). .SH DESCRIPTION .ft R -This routine may be used to determine whether MPI_Init has been called. It is the only routine that may be called before MPI_Init is called. +This routine may be used to determine whether MPI has been +initialized. It is one of a small number of routines that may be +called before MPI is initialized and after MPI has been finalized +(MPI_Finalized is another). .SH ERRORS Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object. @@ -51,6 +54,8 @@ called. By default, this error handler aborts the MPI job, except for I/O functi .SH SEE ALSO .ft R -.sp +.nf MPI_Init - +MPI_Init_thread +MPI_Finalize +MPI_Finalized From c50207c96e7f0a310e75d9bc357916b3d3ebcf86 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Tue, 13 Oct 2015 16:36:17 -0400 Subject: [PATCH 5/5] help-mpi-api.txt: remove now-stale help messages (cherry picked from commit open-mpi/ompi@40b4d5d74dfade903ebd5ed34f0f2bb400f8d716) --- ompi/mpi/help-mpi-api.txt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ompi/mpi/help-mpi-api.txt b/ompi/mpi/help-mpi-api.txt index 6947369a4c..0045d30525 100644 --- a/ompi/mpi/help-mpi-api.txt +++ b/ompi/mpi/help-mpi-api.txt @@ -11,13 +11,6 @@ # # This is the US/English general help file for Open MPI. # -[mpi-function-after-finalize] -Calling any MPI-function after calling MPI_Finalize is erroneous. -The only exceptions are MPI_Initialized, MPI_Finalized and MPI_Get_version. -# -[mpi-initialize-twice] -Calling MPI_Init or MPI_Init_thread twice is erroneous. -# [mpi-abort] MPI_ABORT was invoked on rank %d in communicator %s with errorcode %d.