[WIP] Argobots Support

opal/mca/threads/configure.m4

@@ -25,26 +25,36 @@ m4_define(MCA_opal_threads_CONFIGURE_MODE, STOP_AT_FIRST)
 AC_DEFINE_UNQUOTED([OPAL_ENABLE_MULTI_THREADS], [1],
 	[Whether we should enable thread support within the OPAL code base])
 AC_DEFUN([MCA_opal_threads_CONFIG],[
-        threads_base_include=
-
         # All components look at this value
         AC_ARG_WITH([threads],
             [AC_HELP_STRING([--with-threads=TYPE],
-                        [Build high resolution threads component TYPE])])
+                        [Build high resolution threads component TYPE])],
+                        [],
+                        [with_threads=pthreads])
+
+        thread_type=$with_threads
 
         # first, compile all the components
         MCA_CONFIGURE_FRAMEWORK($1, $2, 1)
 
-        if test "$mutex_base_include" = "" ; then
-            mutex_base_include="pthreads/mutex_unix.h"
-        fi
+        threads_base_include="${thread_type}/threads_${thread_type}_threads.h"
+        mutex_base_include="${thread_type}/threads_${thread_type}_mutex.h"
+        tsd_base_include="${thread_type}/threads_${thread_type}_tsd.h"
+        wait_sync_base_include="${thread_type}/threads_${thread_type}_wait_sync.h"
+
         AC_DEFINE_UNQUOTED([MCA_threads_IMPLEMENTATION_HEADER],
                            ["opal/mca/threads/$threads_base_include"],
                            [Header to include for threads implementation])
 
         AC_DEFINE_UNQUOTED([MCA_mutex_IMPLEMENTATION_HEADER],
                            ["opal/mca/threads/$mutex_base_include"],
                            [Header to include for mutex implementation])
-])
 
+        AC_DEFINE_UNQUOTED([MCA_tsd_IMPLEMENTATION_HEADER],
+                           ["opal/mca/threads/$tsd_base_include"],
+                           [Header to include for tsd implementation])
 
+        AC_DEFINE_UNQUOTED([MCA_wait_sync_IMPLEMENTATION_HEADER],
+                           ["opal/mca/threads/$wait_sync_base_include"],
+                           [Header to include for wait_sync implementation])
+])

opal/mca/threads/mutex.h

@@ -29,6 +29,8 @@
 
 #include "opal_config.h"
 
+BEGIN_C_DECLS
+
 /**
 * @file:
  *
@@ -37,17 +39,14 @@
  * Functions for locking of critical sections.
  */
 
-
 /**
  * Opaque mutex object
  */
 
 typedef struct opal_mutex_t opal_mutex_t;
 typedef struct opal_mutex_t opal_recursive_mutex_t;
 
-BEGIN_C_DECLS
 #include MCA_mutex_IMPLEMENTATION_HEADER
-END_C_DECLS
 
 OBJ_CLASS_DECLARATION(opal_mutex_t);
 OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);

opal/mca/threads/pthreads/Makefile.am

@@ -21,9 +21,12 @@
 noinst_LTLIBRARIES = libmca_threads_pthreads.la
 
 libmca_threads_pthreads_la_SOURCES = \
-    threads_pthreads.h \
     threads_pthreads_component.c \
-    threads_pthreads_mutex.c \
     threads_pthreads_condition.c \
+    threads_pthreads_module.c \
+    threads_pthreads_mutex.c \
+    threads_pthreads_mutex.h \
+    threads_pthreads_threads.h \
+    threads_pthreads_tsd.h \
     threads_pthreads_wait_sync.c \
-    threads_pthreads_module.c
+    threads_pthreads_wait_sync.h

opal/mca/threads/pthreads/configure.m4

@@ -30,11 +30,19 @@ AC_DEFUN([MCA_opal_threads_pthreads_COMPILE_MODE], [
 ])
 
 AC_DEFUN([MCA_opal_threads_pthreads_POST_CONFIG],[
-    AS_IF([test "$1" = "1"], [threads_base_include="pthreads/threads_pthreads.h"])
+    AS_IF([test "$1" = "1"], [threads_base_include="pthreads/threads_pthreads_threads.h"])
 ])dnl
 
 AC_DEFUN([MCA_opal_mutex_pthreads_POST_CONFIG],[
-    AS_IF([test "$1" = "1"], [mutex_base_include="pthreads/mutex_unix.h"])
+    AS_IF([test "$1" = "1"], [mutex_base_include="pthreads/threads_pthreads_mutex.h"])
+])dnl
+
+AC_DEFUN([MCA_opal_tsd_pthreads_POST_CONFIG],[
+    AS_IF([test "$1" = "1"], [threads_base_include="pthreads/threads_pthreads_tsd.h"])
+])dnl
+
+AC_DEFUN([MCA_opal_wait_sync_pthreads_POST_CONFIG],[
+    AS_IF([test "$1" = "1"], [mutex_base_include="pthreads/threads_pthreads_wait_sync.h"])
 ])dnl
 
 # MCA_threads_pthreads_CONFIG(action-if-can-compile,

opal/mca/threads/pthreads/threads_pthreads_component.c

@@ -25,7 +25,6 @@
 
 #include "opal/mca/threads/thread.h"
 #include "opal/mca/threads/threads.h"
-#include "opal/mca/threads/pthreads/threads_pthreads.h"
 #include "opal/constants.h"
 
 static int opal_threads_pthreads_open(void);

opal/mca/threads/pthreads/mutex_unix.h → opal/mca/threads/pthreads/threads_pthreads_mutex.h

@@ -23,8 +23,8 @@
  * $HEADER$
  */
 
-#ifndef  OPAL_MUTEX_UNIX_H
-#define  OPAL_MUTEX_UNIX_H 1
+#ifndef  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_MUTEX_H
+#define  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_MUTEX_H 1
 
 /**
  * @file:
@@ -213,4 +213,4 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
 
 END_C_DECLS
 
-#endif                          /* OPAL_MUTEX_UNIX_H */
+#endif           /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_MUTEX_H */

opal/mca/threads/pthreads/threads_pthreads_threads.h

@@ -0,0 +1,15 @@
+
+#ifndef  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_THREADS_H
+#define  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_THREADS_H 1
+
+#include <pthread.h>
+#include <signal.h>
+
+struct opal_thread_t {
+    opal_object_t super;
+    opal_thread_fn_t t_run;
+    void* t_arg;
+    pthread_t t_handle;
+};
+
+#endif /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_THREADS_H */

opal/mca/threads/pthreads/threads_pthreads_tsd.h

@@ -0,0 +1,29 @@
+
+#ifndef  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_TSD_H
+#define  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_TSD_H 1
+
+#include <pthread.h>
+#include <signal.h>
+
+typedef pthread_key_t opal_tsd_key_t;
+
+static inline int
+opal_tsd_key_delete(opal_tsd_key_t key)
+{
+    return pthread_key_delete(key);
+}
+
+static inline int
+opal_tsd_setspecific(opal_tsd_key_t key, void *value)
+{
+    return pthread_setspecific(key, value);
+}
+
+static inline int
+opal_tsd_getspecific(opal_tsd_key_t key, void **valuep)
+{
+    *valuep = pthread_getspecific(key);
+    return OPAL_SUCCESS;
+}
+
+#endif /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_TSD_H */

opal/mca/threads/pthreads/threads_pthreads_wait_sync.h

@@ -0,0 +1,77 @@
+
+#ifndef  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H
+#define  OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H 1
+
+typedef struct ompi_wait_sync_t {
+    opal_atomic_int32_t count;
+    int32_t status;
+    pthread_cond_t condition;
+    pthread_mutex_t lock;
+    struct ompi_wait_sync_t *next;
+    struct ompi_wait_sync_t *prev;
+    volatile bool signaling;
+} ompi_wait_sync_t;
+
+#define SYNC_WAIT(sync)                 (opal_using_threads() ? ompi_sync_wait_mt (sync) : sync_wait_st (sync))
+
+/* The loop in release handles a race condition between the signaling
+ * thread and the destruction of the condition variable. The signaling
+ * member will be set to false after the final signaling thread has
+ * finished operating on the sync object. This is done to avoid
+ * extra atomics in the signalling function and keep it as fast
+ * as possible. Note that the race window is small so spinning here
+ * is more optimal than sleeping since this macro is called in
+ * the critical path. */
+#define WAIT_SYNC_RELEASE(sync)                       \
+    if (opal_using_threads()) {                       \
+        while ((sync)->signaling) {                   \
+            continue;                                 \
+        }                                             \
+        pthread_cond_destroy(&(sync)->condition);     \
+        pthread_mutex_destroy(&(sync)->lock);         \
+    }
+
+#define WAIT_SYNC_RELEASE_NOWAIT(sync)                \
+    if (opal_using_threads()) {                       \
+        pthread_cond_destroy(&(sync)->condition);     \
+        pthread_mutex_destroy(&(sync)->lock);         \
+    }
+
+
+#define WAIT_SYNC_SIGNAL(sync)                        \
+    if (opal_using_threads()) {                       \
+        pthread_mutex_lock(&(sync->lock));            \
+        pthread_cond_signal(&sync->condition);        \
+        pthread_mutex_unlock(&(sync->lock));          \
+        sync->signaling = false;                      \
+    }
+
+#define WAIT_SYNC_SIGNALLED(sync){                    \
+        (sync)->signaling = false;                    \
+}
+
+OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
+static inline int sync_wait_st (ompi_wait_sync_t *sync)
+{
+    while (sync->count > 0) {
+        opal_progress();
+    }
+
+    return sync->status;
+}
+
+
+#define WAIT_SYNC_INIT(sync,c)                                  \
+    do {                                                        \
+        (sync)->count = (c);                                    \
+        (sync)->next = NULL;                                    \
+        (sync)->prev = NULL;                                    \
+        (sync)->status = 0;                                     \
+        (sync)->signaling = (0 != (c));                         \
+        if (opal_using_threads()) {                             \
+            pthread_cond_init (&(sync)->condition, NULL);       \
+            pthread_mutex_init (&(sync)->lock, NULL);           \
+        }                                                       \
+    } while(0)
+
+#endif /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H */

opal/mca/threads/threads.h

@@ -27,9 +27,6 @@
 
 #include "opal_config.h"
 
-#include <pthread.h>
-#include <signal.h>
-
 #include "opal/class/opal_object.h"
 #if OPAL_ENABLE_DEBUG
 #include "opal/util/output.h"
@@ -44,12 +41,7 @@ typedef void *(*opal_thread_fn_t) (opal_object_t *);
 
 #define OPAL_THREAD_CANCELLED   ((void*)1);
 
-struct opal_thread_t {
-    opal_object_t super;
-    opal_thread_fn_t t_run;
-    void* t_arg;
-    pthread_t t_handle;
-};
+#include MCA_threads_IMPLEMENTATION_HEADER
 
 typedef struct opal_thread_t opal_thread_t;
 
@@ -140,7 +132,6 @@ OPAL_DECLSPEC void opal_thread_kill(opal_thread_t *, int sig);
 OPAL_DECLSPEC void opal_thread_set_main(void);
 OPAL_DECLSPEC void opal_event_use_threads(void);
 
-
 END_C_DECLS
 
 #endif /* OPAL_MCA_THREAD_H */