btl/uct: put active tls in a list on the mca_btl_uct_md_t structure

This simplifies the code a bit by moving mca_btl_uct_tl_t ownership to the
mca_btl_uct_md_t class.

Author: hjelmn

opal/mca/btl/uct/btl_uct_component.c

@@ -500,8 +500,10 @@ static int mca_btl_uct_component_process_uct_md(uct_md_resource_desc_t *md_desc)
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
+    /* module will take ownership of the md */
     module = mca_btl_uct_alloc_module(md, md_attr.rkey_packed_size);
     if (NULL == module) {
+        OBJ_RELEASE(md);
         uct_release_tl_resource_list(tl_desc);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -512,10 +514,6 @@ static int mca_btl_uct_component_process_uct_md(uct_md_resource_desc_t *md_desc)
 
     uct_release_tl_resource_list(tl_desc);
 
-    /* release the initial reference to the md object. if any modules were created the UCT md will
-     * remain open until those modules are finalized. */
-    OBJ_RELEASE(md);
-
     if (NULL == module->am_tl && NULL == module->rdma_tl && (NULL == module->conn_tl || !consider_for_connection_module)) {
         BTL_VERBOSE(("uct memory domain %s does not have any appropriate tls", md->md_name));
         mca_btl_uct_finalize(&module->super);
@@ -842,10 +840,9 @@ static int mca_btl_uct_component_progress(void)
 
         /* unlike ucp, uct actually tells us something useful! its almost like it was "inspired"
          * by the btl progress functions.... */
-        ret += mca_btl_uct_tl_progress(module->rdma_tl, starting_index);
-
-        if (module->am_tl != module->rdma_tl) {
-            ret += mca_btl_uct_tl_progress(module->am_tl, starting_index);
+        mca_btl_uct_tl_t *tl;
+        OPAL_LIST_FOREACH(tl, &module->md->tls, mca_btl_uct_tl_t) {
+            ret += mca_btl_uct_tl_progress(tl, starting_index);
         }
 
         mca_btl_uct_component_progress_connections (module->conn_tl);

opal/mca/btl/uct/btl_uct_module.c

@@ -289,19 +289,8 @@ int mca_btl_uct_finalize(mca_btl_base_module_t *btl)
         mca_rcache_base_module_destroy(uct_module->rcache);
     }
 
-    if (NULL != uct_module->am_tl) {
-        OBJ_RELEASE(uct_module->am_tl);
-    }
-
-    if (NULL != uct_module->conn_tl) {
-        OBJ_RELEASE(uct_module->conn_tl);
-    }
-
-    if (NULL != uct_module->rdma_tl) {
-        OBJ_RELEASE(uct_module->rdma_tl);
-    }
-
     OBJ_DESTRUCT(&uct_module->endpoint_lock);
+    OBJ_RELEASE(uct_module->md);
 
     free(uct_module);
 
@@ -357,10 +346,13 @@ static void mca_btl_uct_md_construct(mca_btl_uct_md_t *md)
 {
     md->uct_md = NULL;
     md->md_name = NULL;
+    OBJ_CONSTRUCT(&md->tls, opal_list_t);
 }
 
 static void mca_btl_uct_md_destruct(mca_btl_uct_md_t *md)
 {
+    OPAL_LIST_DESTRUCT(&md->tls);
+
     free(md->md_name);
     if (md->uct_md) {
         uct_md_close(md->uct_md);

opal/mca/btl/uct/btl_uct_tl.c

@@ -161,10 +161,6 @@ static void mca_btl_uct_tl_destructor(mca_btl_uct_tl_t *tl)
         ucs_async_context_destroy(tl->ucs_async);
     }
 
-    if (tl->uct_md) {
-        OBJ_RELEASE(tl->uct_md);
-    }
-
     free(tl->uct_tl_name);
     free(tl->uct_dev_name);
 
@@ -436,7 +432,6 @@ static mca_btl_uct_tl_t *mca_btl_uct_create_tl(mca_btl_uct_module_t *module, mca
 
     /* initialize btl tl structure */
     tl->uct_md = md;
-    OBJ_RETAIN(md);
 
     tl->uct_tl_name = strdup(tl_desc->tl_name);
     tl->uct_dev_name = strdup(tl_desc->dev_name);
@@ -457,7 +452,7 @@ static mca_btl_uct_tl_t *mca_btl_uct_create_tl(mca_btl_uct_module_t *module, mca
         return NULL;
     }
 
-    BTL_VERBOSE(("Interface CAPS for tl %s::%s: 0x%lx", module->md->md_name, tl_desc->tl_name,
+    BTL_VERBOSE(("Interface CAPS for tl %s::%s: 0x%lx", md->md_name, tl_desc->tl_name,
                  (unsigned long) tl->uct_iface_attr.cap.flags));
 
     return tl;
@@ -489,7 +484,6 @@ static void mca_btl_uct_set_tl_rdma(mca_btl_uct_module_t *module, mca_btl_uct_tl
                                                              .cap.put.max_bcopy;
 
     module->rdma_tl = tl;
-    OBJ_RETAIN(tl);
 
     tl->tl_index = (module->am_tl && tl != module->am_tl) ? 1 : 0;
     module->comm_tls[tl->tl_index] = tl;
@@ -502,7 +496,6 @@ static void mca_btl_uct_set_tl_am(mca_btl_uct_module_t *module, mca_btl_uct_tl_t
 {
     BTL_VERBOSE(("tl %s is suitable for active-messaging", tl->uct_tl_name));
     module->am_tl = tl;
-    OBJ_RETAIN(tl);
 
     tl->tl_index = (module->rdma_tl && tl != module->rdma_tl) ? 1 : 0;
     module->comm_tls[tl->tl_index] = tl;
@@ -532,8 +525,6 @@ static int mca_btl_uct_set_tl_conn(mca_btl_uct_module_t *module, mca_btl_uct_tl_
         return rc;
     }
 
-    OBJ_RETAIN(tl);
-
     if (!tl->max_device_contexts) {
         /* if a tl is only being used to create connections do not bother with multiple
          * contexts */
@@ -591,9 +582,6 @@ int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
                           bool evaluate_for_conn_only)
 {
     mca_btl_uct_tl_t *tl;
-    opal_list_t tl_list;
-
-    OBJ_CONSTRUCT(&tl_list, opal_list_t);
 
     for (unsigned i = 0; i < tl_count; ++i) {
         int priority = 0;
@@ -623,28 +611,27 @@ int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
             if (mca_btl_uct_tl_supports_conn(tl) && evaluate_for_conn_only) {
                 BTL_VERBOSE(("evaluating tl %s for forming connections", tl_descs[i].tl_name));
                 int rc = mca_btl_uct_set_tl_conn(module, tl);
-                OBJ_RELEASE(tl);
 
                 if (OPAL_SUCCESS == rc) {
+                    opal_list_append(&md->tls, &tl->super);
                     return OPAL_SUCCESS;
                 }
 
                 BTL_VERBOSE(("tl %s cannot be used for forming connections", tl_descs[i].tl_name));
             } else {
-                opal_list_append(&tl_list, &tl->super);
+                opal_list_append(&md->tls, &tl->super);
             }
         }
     }
 
-    if (0 == opal_list_get_size(&tl_list)) {
+    if (0 == opal_list_get_size(&md->tls)) {
         BTL_VERBOSE(("no suitable tls match filter: %s", mca_btl_uct_component.allowed_transports));
-        OBJ_DESTRUCT(&tl_list);
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
-    opal_list_sort(&tl_list, tl_compare);
+    opal_list_sort(&md->tls, tl_compare);
 
-    OPAL_LIST_FOREACH (tl, &tl_list, mca_btl_uct_tl_t) {
+    OPAL_LIST_FOREACH (tl, &md->tls, mca_btl_uct_tl_t) {
         mca_btl_uct_evaluate_tl(module, tl);
         if (NULL != module->am_tl && NULL != module->rdma_tl
             && (NULL != module->conn_tl
@@ -676,15 +663,21 @@ int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
         module->super.btl_free = NULL;
     }
 
-    OPAL_LIST_DESTRUCT(&tl_list);
-
     if (!(NULL != module->am_tl && mca_btl_uct_tl_requires_connection_tl(module->am_tl))
         && !(NULL != module->rdma_tl && mca_btl_uct_tl_requires_connection_tl(module->rdma_tl))
         && module->conn_tl) {
         /* no connection tl needed for selected transports */
-        OBJ_RELEASE(module->conn_tl);
         module->conn_tl = NULL;
     }
 
+    /* clear out unused tls */
+    mca_btl_uct_tl_t *next;
+    OPAL_LIST_FOREACH_SAFE(tl, next, &md->tls, mca_btl_uct_tl_t) {
+        if (tl != module->conn_tl && tl != module->rdma_tl && tl != module->am_tl) {
+            opal_list_remove_item(&md->tls, &tl->super);
+            OBJ_RELEASE(tl);
+        }
+    }
+
     return OPAL_SUCCESS;
 }

opal/mca/btl/uct/btl_uct_types.h

@@ -70,6 +70,9 @@ struct mca_btl_uct_md_t {
     /** name of the memory domain backing this module */
     char *md_name;
 
+    /** list of mca_btl_uct_tl_t's for this memory domain */
+    opal_list_t tls;
+
     /** UCT memory domain handle */
     uct_md_h uct_md;
 };
@@ -314,7 +317,7 @@ struct mca_btl_uct_tl_t {
     /** relative priority 0 == highest */
     int priority;
 
-    /** memory domain associated with this tl */
+    /** memory domain associated with this tl (no reference) */
     mca_btl_uct_md_t *uct_md;
 
     /** lock protecting tl structures */