Add opal_recursive_mutex_t support to qthreads

Signed-off-by: Jan Ciesko <jciesko@sandia.gov>

(cherry picked from commit 8c499a3)

Author: Jan Ciesko (-EXP)

opal/mca/threads/qthreads/threads_qthreads_mutex.c

@@ -25,8 +25,8 @@
 
 #include "opal_config.h"
 
-#include "opal/constants.h"
 #include "opal/mca/threads/mutex.h"
+#include "opal/constants.h"
 
 /*
  * Wait and see if some upper layer wants to use threads, if support
@@ -36,23 +36,36 @@ bool opal_uses_threads = false;
 
 static void opal_mutex_construct(opal_mutex_t *m)
 {
-    opal_threads_ensure_init_qthreads();
-    opal_mutex_create(m);
+     opal_mutex_create(m);
 }
 
 static void opal_mutex_destruct(opal_mutex_t *m)
 {
 }
 
-OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, opal_mutex_construct, opal_mutex_destruct);
+static void opal_recursive_mutex_construct(opal_mutex_t *m)
+{
+    opal_mutex_recursive_create(m);
+}
 
-static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m)
+static void opal_recursive_mutex_destruct(opal_mutex_t *m)
 {
 }
 
-OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t, opal_recursive_mutex_construct,
+OBJ_CLASS_INSTANCE(opal_mutex_t,
+                   opal_object_t,
+                   opal_mutex_construct,
                    opal_mutex_destruct);
 
+static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m)
+{
+}
+
+OBJ_CLASS_INSTANCE(opal_recursive_mutex_t,
+                   opal_object_t,
+                   opal_recursive_mutex_construct,
+                   opal_recursive_mutex_destruct);
+
 int opal_cond_init(opal_cond_t *cond)
 {
     opal_atomic_lock_init(&cond->m_lock, 0);
@@ -63,7 +76,7 @@ int opal_cond_init(opal_cond_t *cond)
 
 typedef struct {
     int m_signaled;
-    void *m_prev;
+    void * m_prev;
 } cond_waiter_t;
 
 int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
@@ -74,11 +87,11 @@ int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
     opal_atomic_lock(&cond->m_lock);
     cond_waiter_t waiter = {0, NULL};
     if (NULL == cond->m_waiter_head) {
-        cond->m_waiter_tail = (void *) &waiter;
+        cond->m_waiter_tail = (void *)&waiter;
     } else {
-        ((cond_waiter_t *) cond->m_waiter_head)->m_prev = (void *) &waiter;
+        ((cond_waiter_t *)cond->m_waiter_head)->m_prev = (void *)&waiter;
     }
-    cond->m_waiter_head = (void *) &waiter;
+    cond->m_waiter_head = (void *)&waiter;
     opal_atomic_unlock(&cond->m_lock);
 
     while (1) {
@@ -101,7 +114,7 @@ int opal_cond_broadcast(opal_cond_t *cond)
 {
     opal_atomic_lock(&cond->m_lock);
     while (NULL != cond->m_waiter_tail) {
-        cond_waiter_t *p_cur_tail = (cond_waiter_t *) cond->m_waiter_tail;
+        cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail;
         cond->m_waiter_tail = p_cur_tail->m_prev;
         /* Awaken one of threads in a FIFO manner. */
         p_cur_tail->m_signaled = 1;
@@ -116,7 +129,7 @@ int opal_cond_signal(opal_cond_t *cond)
 {
     opal_atomic_lock(&cond->m_lock);
     if (NULL != cond->m_waiter_tail) {
-        cond_waiter_t *p_cur_tail = (cond_waiter_t *) cond->m_waiter_tail;
+        cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail;
         cond->m_waiter_tail = p_cur_tail->m_prev;
         /* Awaken one of threads. */
         p_cur_tail->m_signaled = 1;

opal/mca/threads/qthreads/threads_qthreads_mutex.h

@@ -98,6 +98,8 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
 
 static inline void opal_mutex_create(struct opal_mutex_t *m)
 {
+    opal_threads_ensure_init_qthreads();
+
     opal_atomic_lock_init(&m->m_lock, 0);
     opal_atomic_lock_init(&m->m_lock_atomic, 0);
     m->m_recursive = 0;
@@ -108,9 +110,24 @@ static inline void opal_mutex_create(struct opal_mutex_t *m)
 #endif
 }
 
+static inline void opal_mutex_recursive_create(struct opal_mutex_t *m)
+{
+    opal_threads_ensure_init_qthreads();
+
+    opal_atomic_lock_init(&m->m_lock, 0);
+    opal_atomic_lock_init(&m->m_lock_atomic, 0);
+    m->m_recursive = 1;
+#if OPAL_ENABLE_DEBUG
+    m->m_lock_debug = 0;
+    m->m_lock_file = NULL;
+    m->m_lock_line = 0;
+#endif
+}
+
 static inline int opal_mutex_trylock(opal_mutex_t *m)
 {
     opal_threads_ensure_init_qthreads();
+
     int ret = opal_atomic_trylock(&m->m_lock);
     if (0 != ret) {
         /* Yield to avoid a deadlock. */
@@ -122,6 +139,7 @@ static inline int opal_mutex_trylock(opal_mutex_t *m)
 static inline void opal_mutex_lock(opal_mutex_t *m)
 {
     opal_threads_ensure_init_qthreads();
+
     int ret = opal_atomic_trylock(&m->m_lock);
     while (0 != ret) {
         qthread_yield();