v2.0.0: init finalize mutex plus mutex static initializer

ompi/mpi/c/finalized.c

@@ -9,6 +9,9 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,6 +41,14 @@ int MPI_Finalized(int *flag)
 {
     MPI_Comm null = NULL;
 
+    /* We must obtain the lock to guarnatee consistent values of
+       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
+       this lock is held for the bulk of the duration of
+       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
+       lock, we are guaranteed that some other thread is not part way
+       through initialization or finalization. */
+    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
 
@@ -46,17 +57,19 @@ int MPI_Finalized(int *flag)
                MPI_Finalize) or not */
 
             if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG,
                                               FUNC_NAME);
             }
         }
     }
 
-    /* Pretty simple */
-
     *flag = ompi_mpi_finalized;
+    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+
     return MPI_SUCCESS;
 }

ompi/mpi/c/init.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -46,25 +46,6 @@ int MPI_Init(int *argc, char ***argv)
     char *env;
     int required = MPI_THREAD_SINGLE;
 
-    /* Ensure that we were not already initialized or finalized */
-
-    if (ompi_mpi_finalized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt",
-                           "mpi-function-after-finalize", true, FUNC_NAME);
-        }
-        return ompi_errhandler_invoke(NULL, NULL,
-                                      OMPI_ERRHANDLER_TYPE_COMM,
-                                      MPI_ERR_OTHER, FUNC_NAME);
-    } else if (ompi_mpi_initialized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-initialize-twice",
-                           true, FUNC_NAME);
-        }
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER,
-                                      FUNC_NAME);
-    }
-
     /* check for environment overrides for required thread level.  If
        there is, check to see that it is a valid/supported thread level.
        If not, default to MPI_THREAD_MULTIPLE. */

ompi/mpi/c/init_thread.c

@@ -10,6 +10,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -59,23 +62,6 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
     *provided = MPI_THREAD_SINGLE;
 #endif
 
-    /* Ensure that we were not already initialized or finalized */
-
-    if (ompi_mpi_finalized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-function-after-finalize",
-                           true, FUNC_NAME);
-        }
-        return ompi_errhandler_invoke(NULL, NULL, OMPI_ERRHANDLER_TYPE_COMM,
-                                      MPI_ERR_OTHER, FUNC_NAME);
-    } else if (ompi_mpi_initialized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-initialize-twice",
-                           true, FUNC_NAME);
-        }
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, FUNC_NAME);
-    }
-
     /* Call the back-end initialization function (we need to put as
        little in this function as possible so that if it's profiled, we
        don't lose anything) */

ompi/mpi/c/initialized.c

@@ -9,6 +9,9 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,6 +41,14 @@ int MPI_Initialized(int *flag)
 {
     MPI_Comm null = NULL;
 
+    /* We must obtain the lock to guarnatee consistent values of
+       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
+       this lock is held for the bulk of the duration of
+       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
+       lock, we are guaranteed that some other thread is not part way
+       through initialization or finalization. */
+    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
 
@@ -46,17 +57,19 @@ int MPI_Initialized(int *flag)
                MPI_Finalize) or not */
 
             if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG,
                                               FUNC_NAME);
             }
         }
     }
 
-    /* Pretty simple */
-
     *flag = ompi_mpi_initialized;
+    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+
     return MPI_SUCCESS;
 }

ompi/mpi/help-mpi-api.txt

@@ -11,13 +11,6 @@
 #
 # This is the US/English general help file for Open MPI.
 #
-[mpi-function-after-finalize]
-Calling any MPI-function after calling MPI_Finalize is erroneous.
-The only exceptions are MPI_Initialized, MPI_Finalized and MPI_Get_version.
-#
-[mpi-initialize-twice]
-Calling MPI_Init or MPI_Init_thread twice is erroneous.
-#
 [mpi-abort]
 MPI_ABORT was invoked on rank %d in communicator %s
 with errorcode %d.

ompi/mpi/man/man3/MPI_Finalize.3in

@@ -1,5 +1,5 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
@@ -55,3 +55,10 @@ Before the error value is returned, the current MPI error handler is
 called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
 
 
+.SH SEE ALSO
+.ft R
+.nf
+MPI_Init
+MPI_Init_thread
+MPI_Initialized
+MPI_Finalized

ompi/mpi/man/man3/MPI_Finalized.3in

@@ -1,11 +1,11 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
 .TH MPI_Finalized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .SH NAME
-\fBMPI_Finalized \fP \- Checks whether MPI_Finalize has completed.
+\fBMPI_Finalized \fP \- Checks whether MPI has been finalized
 
 .SH SYNTAX
 .ft R
@@ -33,19 +33,29 @@ bool MPI::Is_finalized()
 .ft R
 .TP 1i
 flag
-True if MPI was finalized (logical).
+True if MPI was finalized, and false otherwise (logical).
 .ft R
 .TP 1i
 IERROR
 Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine returns "true" if MPI_Finalize has completed. You can call MPI_Finalized before MPI_Init and after MPI_Finalize.
+This routine may be used to determine whether MPI has been finalized.
+It is one of a small number of routines that may be called before MPI
+is initialized and after MPI has been finalized (MPI_Initialized is
+another).
 
 .SH ERRORS
 Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object.
 .sp
 Before the error value is returned, the current MPI error handler is
 called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
 
+.SH SEE ALSO
+.ft R
+.nf
+MPI_Init
+MPI_Init_thread
+MPI_Initialized
+MPI_Finalize

ompi/mpi/man/man3/MPI_Init.3in

@@ -1,5 +1,5 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
@@ -48,10 +48,10 @@ Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine, or MPI_Init_thread, must be called before any other MPI
-routine (apart from MPI_Initialized) is called. MPI can be initialized
-at most once; subsequent calls to MPI_Init or MPI_Init_thread are
-erroneous.
+This routine, or MPI_Init_thread, must be called before most other MPI
+routines are called.  There are a small number of exceptions, such as
+MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
+once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
 .sp
 All MPI programs must contain a call to MPI_Init or
 MPI_Init_thread. Open MPI accepts the C/C++ \fIargc\fP and \fIargv\fP
@@ -102,4 +102,4 @@ See the MPI man page for a full list of MPI error codes.
 MPI_Init_thread
 MPI_Initialized
 MPI_Finalize
-
+MPI_Finalized

ompi/mpi/man/man3/MPI_Init_thread.3in

@@ -1,7 +1,7 @@
 .\" -*- nroff -*-
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" $COPYRIGHT$
 .TH MPI_Init_thread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .
@@ -57,8 +57,9 @@ Fortran only: Error status (integer).
 .
 .SH DESCRIPTION
 .ft R
-This routine, or MPI_Init, must be called before any other MPI routine
-(apart from MPI_Initialized) is called. MPI can be initialized at most
+This routine, or MPI_Init, must be called before most other MPI
+routines are called. There are a small number of exceptions, such as
+MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
 once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
 .sp
 MPI_Init_thread, as compared to MPI_Init, has a provision to request a
@@ -193,4 +194,4 @@ guarantee that an MPI program can continue past an error.
 MPI_Init
 MPI_Initialized
 MPI_Finalize
-
+MPI_Finalized

ompi/mpi/man/man3/MPI_Initialized.3in

@@ -1,11 +1,11 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
 .TH MPI_Initialized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .SH NAME
-\fBMPI_Initialized\fP \- Indicates whether MPI_Init has been called.
+\fBMPI_Initialized\fP \- Checks whether MPI has been initialized
 
 .SH SYNTAX
 .ft R
@@ -33,15 +33,18 @@ bool Is_initialized()
 .ft R
 .TP 1i
 flag
-Flag is true if MPI_Init has been called and false otherwise.
+True if MPI has been initialized, and false otherwise (logical).
 .ft R
 .TP 1i
 IERROR
 Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine may be used to determine whether MPI_Init has been called. It is the only routine that may be called before MPI_Init is called.
+This routine may be used to determine whether MPI has been
+initialized.  It is one of a small number of routines that may be
+called before MPI is initialized and after MPI has been finalized
+(MPI_Finalized is another).
 
 .SH ERRORS
 Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object.
@@ -51,6 +54,8 @@ called. By default, this error handler aborts the MPI job, except for I/O functi
 
 .SH SEE ALSO
 .ft R
-.sp
+.nf
 MPI_Init
-
+MPI_Init_thread
+MPI_Finalize
+MPI_Finalized

ompi/mpi/tool/init_thread.c

@@ -32,18 +32,8 @@ extern volatile int32_t initted;
 
 int MPI_T_init_thread (int required, int *provided)
 {
-    static volatile int32_t first_init = 1;
     int rc = MPI_SUCCESS;
 
-    if (opal_atomic_cmpset (&first_init, 1, 0) == 1) {
-        OBJ_CONSTRUCT(&mpit_big_lock, opal_mutex_t);
-        initted = 1;
-    }
-
-    while (!initted) {
-        usleep (10);
-    }
-
     mpit_lock ();
 
     do {

ompi/mpi/tool/mpit_common.c

@@ -13,23 +13,18 @@
 
 #include "ompi/mpi/tool/mpit-internal.h"
 
-opal_mutex_t mpit_big_lock = {{0}};
+opal_mutex_t mpit_big_lock = OPAL_MUTEX_STATIC_INIT;
 
 volatile uint32_t mpit_init_count = 0;
-volatile int32_t initted = 0;
 
 void mpit_lock (void)
 {
-    if (initted) {
-        opal_mutex_lock (&mpit_big_lock);
-    }
+    opal_mutex_lock (&mpit_big_lock);
 }
 
 void mpit_unlock (void)
 {
-    if (initted) {
-        opal_mutex_unlock (&mpit_big_lock);
-    }
+    opal_mutex_unlock (&mpit_big_lock);
 }
 
 int ompit_var_type_to_datatype (mca_base_var_type_t type, MPI_Datatype *datatype)