ompi: always enable MPI_THREAD_MULTIPLE support

hjelmn · hjelmn · commit 230d04327e65 · 2016-02-23T10:02:14.000-07:00
This commit removes the --with-mpi-thread-multiple option and forces
MPI_THREAD_MULTIPLE support. This cleans up an abstration violation
in opal where OMPI_ENABLE_THREAD_MULTIPLE determines whether the
opal_using_threads is meaningful. To reduce the performance hit on
MPI_THREAD_SINGLE programs an OPAL_UNLIKELY is used for the
check on opal_using_threads in OPAL_THREAD_* macros.

This commit does not clean up the arguments to the various functions
that take whether muti-threading support is enabled. That should be
done at a later time.

Signed-off-by: Nathan Hjelm &lt;hjelmn@lanl.gov&gt;
diff --git a/config/ompi_config_threads.m4 b/config/ompi_config_threads.m4
diff --git a/configure.ac b/configure.ac
@@ -896,7 +896,6 @@ OPAL_CHECK_BROKEN_QSORT
 # Check out what thread support we have
 #
 OPAL_CONFIG_THREADS
-OMPI_CONFIG_THREADS
 
 CFLAGS="$CFLAGS $THREAD_CFLAGS"
 CPPFLAGS="$CPPFLAGS $THREAD_CPPFLAGS"
diff --git a/ompi/communicator/comm_cid.c b/ompi/communicator/comm_cid.c
@@ -514,7 +514,6 @@ static int ompi_comm_register_cid (uint32_t cid)
                 if ( regcom->cid > cid ) {
                     break;
                 }
-#if OMPI_ENABLE_THREAD_MULTIPLE
                 if( regcom->cid == cid ) {
                     /**
                      * The MPI standard state that is the user responsability to
@@ -527,7 +526,6 @@ static int ompi_comm_register_cid (uint32_t cid)
                     ok = false;
                     break;
                 }
-#endif  /* OMPI_ENABLE_THREAD_MULTIPLE */
             }
             if (ok) {
                 opal_list_insert_pos (&ompi_registered_comms, (opal_list_item_t *) regcom,
diff --git a/ompi/mca/bml/r2/bml_r2_ft.c b/ompi/mca/bml/r2/bml_r2_ft.c
@@ -170,8 +170,7 @@ int mca_bml_r2_ft_event(int state)
              * This will cause the BTL components to discover the available
              * network options on this machine, and post proper modex informaiton.
              */
-            if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS,
-                                                           OMPI_ENABLE_THREAD_MULTIPLE) ) ) {
+            if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1) ) ) {
                 opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
                 return ret;
             }
@@ -257,8 +256,7 @@ int mca_bml_r2_ft_event(int state)
          * This will cause the BTL components to discover the available
          * network options on this machine, and post proper modex informaiton.
          */
-        if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS,
-                                                       OMPI_ENABLE_THREAD_MULTIPLE) ) ) {
+        if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1) ) ) {
             opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
             return ret;
         }
diff --git a/ompi/mca/io/base/io_base_file_select.c b/ompi/mca/io/base/io_base_file_select.c
@@ -215,23 +215,19 @@ int mca_io_base_file_select(ompi_file_t *file,
         }
 
         if (OMPI_SUCCESS !=
-            (ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                              OMPI_ENABLE_THREAD_MULTIPLE))) {
+            (ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
         if (OMPI_SUCCESS !=
-            (ret = mca_fcoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                                 OMPI_ENABLE_THREAD_MULTIPLE))) {
+            (ret = mca_fcoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
         if (OMPI_SUCCESS !=
-            (ret = mca_fbtl_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                                OMPI_ENABLE_THREAD_MULTIPLE))) {
+            (ret = mca_fbtl_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
         if (OMPI_SUCCESS !=
-            (ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                                    OMPI_ENABLE_THREAD_MULTIPLE))) {
+            (ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
     }
diff --git a/ompi/mca/io/base/io_base_frame.c b/ompi/mca/io/base/io_base_frame.c
@@ -38,7 +38,7 @@ static int mca_io_base_open(mca_base_open_flag_t flags)
         return ret;
     }
 
-    return mca_io_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, OMPI_ENABLE_THREAD_MULTIPLE);
+    return mca_io_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1);
 }
 
 MCA_BASE_FRAMEWORK_DECLARE(ompi, io, "I/O", NULL, mca_io_base_open, NULL,
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c
@@ -43,7 +43,6 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev,
 
     switch (ev->type) {
         case PTL_EVENT_AUTO_FREE:
-#if OMPI_ENABLE_THREAD_MULTIPLE
             OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex);
             switch (block->status) {
                 case BLOCK_STATUS_ACTIVATED: /* May be encountered with multi threading */
@@ -73,19 +72,10 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev,
                                         __FILE__, __LINE__, block->status);
                     break;
             }
-#else
-            if (OPAL_UNLIKELY(block->release_on_free)) {
-                opal_list_remove_item(&ompi_mtl_portals4.recv_short_blocks,
-                                      &block->base);
-                ompi_mtl_portals4_recv_short_block_free(block);
-            } else {
-                ompi_mtl_portals4_activate_block(block);
-            }
-#endif
+
             break;
 
         case PTL_EVENT_AUTO_UNLINK:
-#if OMPI_ENABLE_THREAD_MULTIPLE
             OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex);
             switch (block->status) {
                 case BLOCK_STATUS_ACTIVATED: /* Normal case */
@@ -115,14 +105,10 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev,
                                         __FILE__, __LINE__, block->status);
                     break;
             }
-#else
-            block->status = BLOCK_STATUS_WAITING_FREE;
-            ompi_mtl_portals4.active_recv_short_blocks--;
-#endif
+
             break;
 
         case PTL_EVENT_LINK:
-#if OMPI_ENABLE_THREAD_MULTIPLE
             OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex);
             switch (block->status) {
                 case BLOCK_STATUS_WAITING_LINK:
@@ -138,10 +124,7 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev,
                                         __FILE__, __LINE__, block->status);
                     break;
             }
-#else
-            block->status = BLOCK_STATUS_ACTIVATED;
-            ompi_mtl_portals4.active_recv_short_blocks++;
-#endif
+
             break;
 
         default:
diff --git a/ompi/mca/pml/ob1/pml_ob1_irecv.c b/ompi/mca/pml/ob1/pml_ob1_irecv.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
@@ -101,16 +101,16 @@ int mca_pml_ob1_recv(void *addr,
     mca_pml_ob1_recv_request_t *recvreq = NULL;
     int rc;
 
-#if !OMPI_ENABLE_THREAD_MULTIPLE
-    recvreq = mca_pml_ob1_recvreq;
-    mca_pml_ob1_recvreq = NULL;
-    if( OPAL_UNLIKELY(NULL == recvreq) )
-#endif  /* !OMPI_ENABLE_THREAD_MULTIPLE */
-        {
-            MCA_PML_OB1_RECV_REQUEST_ALLOC(recvreq);
-            if (NULL == recvreq)
-                return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-        }
+    if (OPAL_LIKELY(!ompi_mpi_thread_multiple)) {
+        recvreq = mca_pml_ob1_recvreq;
+        mca_pml_ob1_recvreq = NULL;
+    }
+
+    if( OPAL_UNLIKELY(NULL == recvreq) ) {
+        MCA_PML_OB1_RECV_REQUEST_ALLOC(recvreq);
+        if (NULL == recvreq)
+            return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+    }
 
     MCA_PML_OB1_RECV_REQUEST_INIT(recvreq, addr, count, datatype,
                                   src, tag, comm, false);
@@ -128,16 +128,12 @@ int mca_pml_ob1_recv(void *addr,
 
     rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
 
-#if OMPI_ENABLE_THREAD_MULTIPLE
-    MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
-#else
-    if( NULL != mca_pml_ob1_recvreq ) {
+    if (OPAL_UNLIKELY(ompi_mpi_thread_multiple || NULL != mca_pml_ob1_recvreq)) {
         MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
     } else {
         mca_pml_ob1_recv_request_fini (recvreq);
         mca_pml_ob1_recvreq = recvreq;
     }
-#endif
 
     return rc;
 }
diff --git a/ompi/mca/pml/ob1/pml_ob1_isend.c b/ompi/mca/pml/ob1/pml_ob1_isend.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -222,16 +222,17 @@ int mca_pml_ob1_send(const void *buf,
         }
     }
 
-#if !OMPI_ENABLE_THREAD_MULTIPLE
-    sendreq = mca_pml_ob1_sendreq;
-    mca_pml_ob1_sendreq = NULL;
-    if( OPAL_UNLIKELY(NULL == sendreq) )
-#endif  /* !OMPI_ENABLE_THREAD_MULTIPLE */
-        {
-            MCA_PML_OB1_SEND_REQUEST_ALLOC(comm, dst, sendreq);
-            if (NULL == sendreq)
-                return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-        }
+    if (OPAL_LIKELY(!ompi_mpi_thread_multiple)) {
+        sendreq = mca_pml_ob1_sendreq;
+        mca_pml_ob1_sendreq = NULL;
+    }
+
+    if( OPAL_UNLIKELY(NULL == sendreq) ) {
+        MCA_PML_OB1_SEND_REQUEST_ALLOC(comm, dst, sendreq);
+        if (NULL == sendreq)
+            return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+    }
+
     sendreq->req_send.req_base.req_proc = dst_proc;
     sendreq->rdma_frag = NULL;
 
@@ -253,16 +254,12 @@ int mca_pml_ob1_send(const void *buf,
         rc = sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR;
     }
 
-#if OMPI_ENABLE_THREAD_MULTIPLE
-    MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
-#else
-    if( NULL != mca_pml_ob1_sendreq ) {
+    if (OPAL_UNLIKELY(ompi_mpi_thread_multiple || NULL != mca_pml_ob1_sendreq)) {
         MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
     } else {
         mca_pml_ob1_send_request_fini (sendreq);
         mca_pml_ob1_sendreq = sendreq;
     }
-#endif
 
     return rc;
 }
diff --git a/ompi/mca/pml/v/pml_v_component.c b/ompi/mca/pml/v/pml_v_component.c
@@ -56,7 +56,7 @@ mca_pml_base_component_2_0_0_t mca_pml_v_component =
 };
 
 static bool pml_v_enable_progress_treads = OPAL_ENABLE_PROGRESS_THREADS;
-static bool pml_v_enable_mpi_thread_multiple = OMPI_ENABLE_THREAD_MULTIPLE;
+static bool pml_v_enable_mpi_thread_multiple = 1;
 
 static char *ompi_pml_vprotocol_include_list;
 static char *ompi_pml_v_output;
diff --git a/ompi/mca/topo/base/topo_base_lazy_init.c b/ompi/mca/topo/base/topo_base_lazy_init.c
@@ -49,8 +49,7 @@ int mca_topo_base_lazy_init(void)
         }
 
         if (OMPI_SUCCESS !=
-            (err = mca_topo_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                                OMPI_ENABLE_THREAD_MULTIPLE))) {
+            (err = mca_topo_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
             return err;
         }
     }
diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -13,6 +14,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -50,16 +53,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
         }
     }
 
-    /*
-     *   A thread compliant MPI implementation will be able to return provided
-     *   = MPI_THREAD_MULTIPLE. Such an implementation may always return provided
-     *   = MPI_THREAD_MULTIPLE, irrespective of the value of required.
-     */
-#if OMPI_ENABLE_THREAD_MULTIPLE
-    *provided = MPI_THREAD_MULTIPLE;
-#else
-    *provided = MPI_THREAD_SINGLE;
-#endif
+    *provided = required;
 
     /* Call the back-end initialization function (we need to put as
        little in this function as possible so that if it's profiled, we
diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c
@@ -325,15 +325,7 @@ void ompi_mpi_thread_level(int requested, int *provided)
      */
     ompi_mpi_thread_requested = requested;
 
-    if (OMPI_ENABLE_THREAD_MULTIPLE == 1) {
-        ompi_mpi_thread_provided = *provided = requested;
-    } else {
-        if (MPI_THREAD_MULTIPLE == requested) {
-            ompi_mpi_thread_provided = *provided = MPI_THREAD_SERIALIZED;
-        } else {
-            ompi_mpi_thread_provided = *provided = requested;
-        }
-    }
+    ompi_mpi_thread_provided = *provided = requested;
 
     if (!ompi_mpi_main_thread) {
         ompi_mpi_main_thread = opal_thread_get_self();
@@ -530,15 +522,13 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     memset ( &threadlevel_bf, 0, sizeof(uint8_t));
     OMPI_THREADLEVEL_SET_BITFLAG ( ompi_mpi_thread_provided, threadlevel_bf );
 
-#if OMPI_ENABLE_THREAD_MULTIPLE
     /* add this bitflag to the modex */
     OPAL_MODEX_SEND_STRING(ret, OPAL_PMIX_GLOBAL,
                            "MPI_THREAD_LEVEL", &threadlevel_bf, sizeof(uint8_t));
     if (OPAL_SUCCESS != ret) {
         error = "ompi_mpi_init: modex send thread level";
         goto error;
     }
-#endif
 
     /* If thread support was enabled, then setup OPAL to allow for
        them. */
diff --git a/ompi/tools/ompi_info/param.c b/ompi/tools/ompi_info/param.c
diff --git a/opal/threads/mutex.c b/opal/threads/mutex.c
diff --git a/opal/threads/mutex.h b/opal/threads/mutex.h
diff --git a/oshmem/runtime/oshmem_shmem_init.c b/oshmem/runtime/oshmem_shmem_init.c
diff --git a/oshmem/tools/oshmem_info/param.c b/oshmem/tools/oshmem_info/param.c

Original file line number	Diff line number	Diff line change
`@@ -896,7 +896,6 @@ OPAL_CHECK_BROKEN_QSORT`
`896`	`896`	`# Check out what thread support we have`
`897`	`897`	`#`
`898`	`898`	`OPAL_CONFIG_THREADS`
`899`		`-OMPI_CONFIG_THREADS`
`900`	`899`
`901`	`900`	`CFLAGS="$CFLAGS $THREAD_CFLAGS"`
`902`	`901`	`CPPFLAGS="$CPPFLAGS $THREAD_CPPFLAGS"`
Original file line number	Diff line number	Diff line change
`@@ -514,7 +514,6 @@ static int ompi_comm_register_cid (uint32_t cid)`
`514`	`514`	`if ( regcom->cid > cid ) {`
`515`	`515`	`break;`
`516`	`516`	`}`
`517`		`-#if OMPI_ENABLE_THREAD_MULTIPLE`
`518`	`517`	`if( regcom->cid == cid ) {`
`519`	`518`	`/**`
`520`	`519`	`* The MPI standard state that is the user responsability to`
`@@ -527,7 +526,6 @@ static int ompi_comm_register_cid (uint32_t cid)`
`527`	`526`	`ok = false;`
`528`	`527`	`break;`
`529`	`528`	`}`
`530`		`-#endif /* OMPI_ENABLE_THREAD_MULTIPLE */`
`531`	`529`	`}`
`532`	`530`	`if (ok) {`
`533`	`531`	`opal_list_insert_pos (&ompi_registered_comms, (opal_list_item_t *) regcom,`
Original file line number	Diff line number	Diff line change
`@@ -215,23 +215,19 @@ int mca_io_base_file_select(ompi_file_t *file,`
`215`	`215`	`}`
`216`	`216`
`217`	`217`	`if (OMPI_SUCCESS !=`
`218`		`- (ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,`
`219`		`- OMPI_ENABLE_THREAD_MULTIPLE))) {`
	`218`	`+ (ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {`
`220`	`219`	`return err;`
`221`	`220`	`}`
`222`	`221`	`if (OMPI_SUCCESS !=`
`223`		`- (ret = mca_fcoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,`
`224`		`- OMPI_ENABLE_THREAD_MULTIPLE))) {`
	`222`	`+ (ret = mca_fcoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {`
`225`	`223`	`return err;`
`226`	`224`	`}`
`227`	`225`	`if (OMPI_SUCCESS !=`
`228`		`- (ret = mca_fbtl_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,`
`229`		`- OMPI_ENABLE_THREAD_MULTIPLE))) {`
	`226`	`+ (ret = mca_fbtl_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {`
`230`	`227`	`return err;`
`231`	`228`	`}`
`232`	`229`	`if (OMPI_SUCCESS !=`
`233`		`- (ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,`
`234`		`- OMPI_ENABLE_THREAD_MULTIPLE))) {`
	`230`	`+ (ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {`
`235`	`231`	`return err;`
`236`	`232`	`}`
`237`	`233`	`}`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ static int mca_io_base_open(mca_base_open_flag_t flags)`
`38`	`38`	`return ret;`
`39`	`39`	`}`
`40`	`40`
`41`		`- return mca_io_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, OMPI_ENABLE_THREAD_MULTIPLE);`
	`41`	`+ return mca_io_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1);`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`MCA_BASE_FRAMEWORK_DECLARE(ompi, io, "I/O", NULL, mca_io_base_open, NULL,`