btl/uct: move the async context from the module to the tl

There is no real benefit from sharing the async context between tls. Given this
and some other changes that will be made it makes sense to move it from the
module to the tl.

Signed-off-by: Nathan Hjelm <hjelmn@google.com>

Author: hjelmn

opal/mca/btl/uct/btl_uct.h

@@ -79,9 +79,6 @@ struct mca_btl_uct_module_t {
     /** mutex to protect the module */
     opal_recursive_mutex_t lock;
 
-    /** async context */
-    ucs_async_context_t *ucs_async;
-
     /** transport for active messaging */
     mca_btl_uct_tl_t *am_tl;
 

opal/mca/btl/uct/btl_uct_component.c

@@ -393,13 +393,6 @@ static mca_btl_uct_module_t *mca_btl_uct_alloc_module(const char *md_name, mca_b
     module->md_name = strdup(md_name);
     module->super.btl_registration_handle_size = registration_size;
 
-    ucs_status = ucs_async_context_create(UCS_ASYNC_MODE_THREAD, &module->ucs_async);
-    if (UCS_OK != ucs_status) {
-        BTL_VERBOSE(("Could not create a UCT async context"));
-        mca_btl_uct_finalize(&module->super);
-        return NULL;
-    }
-
     return module;
 }
 

opal/mca/btl/uct/btl_uct_module.c

@@ -302,8 +302,6 @@ int mca_btl_uct_finalize(mca_btl_base_module_t *btl)
         OBJ_RELEASE(uct_module->rdma_tl);
     }
 
-    ucs_async_context_destroy(uct_module->ucs_async);
-
     OBJ_DESTRUCT(&uct_module->endpoint_lock);
 
     free(uct_module->md_name);

opal/mca/btl/uct/btl_uct_tl.c

@@ -156,6 +156,10 @@ static void mca_btl_uct_tl_destructor(mca_btl_uct_tl_t *tl)
         }
     }
 
+    if (tl->ucs_async) {
+        ucs_async_context_destroy(tl->ucs_async);
+    }
+
     if (tl->uct_md) {
         OBJ_RELEASE(tl->uct_md);
     }
@@ -291,7 +295,7 @@ static int mca_btl_uct_populate_tl_attr(mca_btl_uct_module_t *module, mca_btl_uc
 
     /* do the bare minimum to get tl attributes */
     uct_worker_h uct_worker;
-    ucs_status = uct_worker_create(module->ucs_async, UCS_THREAD_MODE_SINGLE, &uct_worker);
+    ucs_status = uct_worker_create(tl->ucs_async, UCS_THREAD_MODE_SINGLE, &uct_worker);
     if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
         BTL_VERBOSE(("could not create a UCT worker"));
         return OPAL_ERROR;
@@ -364,7 +368,7 @@ mca_btl_uct_device_context_t *mca_btl_uct_context_create(mca_btl_uct_module_t *m
      * use our own locks just go ahead and use UCS_THREAD_MODE_SINGLE. if they ever fix their
      * api then change this back to UCS_THREAD_MODE_MULTI and remove the locks around the
      * various UCT calls. */
-    ucs_status = uct_worker_create(module->ucs_async, UCS_THREAD_MODE_SINGLE, &context->uct_worker);
+    ucs_status = uct_worker_create(tl->ucs_async, UCS_THREAD_MODE_SINGLE, &context->uct_worker);
     if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
         BTL_VERBOSE(("could not create a UCT worker"));
         mca_btl_uct_context_destroy(context);
@@ -437,6 +441,13 @@ static mca_btl_uct_tl_t *mca_btl_uct_create_tl(mca_btl_uct_module_t *module, mca
 
     (void) uct_md_iface_config_read(md->uct_md, tl_desc->tl_name, NULL, NULL, &tl->uct_tl_config);
 
+    ucs_status_t ucs_status = ucs_async_context_create(UCS_ASYNC_MODE_THREAD, &tl->ucs_async);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("Could not create a UCT async context"));
+        OBJ_RELEASE(tl);
+        return NULL;
+    }
+
     int rc = mca_btl_uct_populate_tl_attr(module, tl);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         OBJ_RELEASE(tl);

opal/mca/btl/uct/btl_uct_types.h

@@ -338,6 +338,9 @@ struct mca_btl_uct_tl_t {
 
     /** interface attributes */
     uct_iface_attr_t uct_iface_attr;
+
+    /** async context */
+    ucs_async_context_t *ucs_async;
 };
 
 typedef struct mca_btl_uct_tl_t mca_btl_uct_tl_t;