ofi: match common and MCA interfaces

Rename initialization calls to match the MCA terminology
and restructure code to match the calling pattern of
normal components (so during component_register, OFI
components should call common_ofi_mca_register).

Signed-off-by: Brian Barrett <bbarrett@amazon.com>

Author: bwbarrett

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -254,9 +254,7 @@ ompi_mtl_ofi_component_register(void)
                                     MCA_BASE_VAR_SCOPE_READONLY,
                                     &ompi_mtl_ofi.num_ofi_contexts);
 
-    opal_common_ofi_register_mca_variables(&mca_mtl_ofi_component.super.mtl_version);
-
-    return OMPI_SUCCESS;
+    return opal_common_ofi_mca_register(&mca_mtl_ofi_component.super.mtl_version);
 }
 
 
@@ -285,7 +283,7 @@ ompi_mtl_ofi_component_open(void)
             "provider_exclude")) {
         return OMPI_ERR_NOT_AVAILABLE;
     }
-    return opal_common_ofi_init();
+    return opal_common_ofi_open();
 }
 
 static int
@@ -302,9 +300,7 @@ ompi_mtl_ofi_component_close(void)
 #if OPAL_CUDA_SUPPORT
     mca_common_cuda_fini();
 #endif
-    opal_common_ofi_mca_deregister();
-    opal_common_ofi_fini();
-    return OMPI_SUCCESS;
+    return opal_common_ofi_close();
 }
 
 int
@@ -582,8 +578,6 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     int universe_size;
     char *univ_size_str;
 
-    opal_common_ofi_mca_register();
-
     opal_output_verbose(1, opal_common_ofi.output,
                         "%s:%d: mtl:ofi:provider_include = \"%s\"\n",
                         __FILE__, __LINE__, *opal_common_ofi.prov_include);

opal/mca/btl/ofi/btl_ofi_component.c

@@ -123,6 +123,7 @@ static int validate_info(struct fi_info *info, uint64_t required_caps, char **in
 /* Register the MCA parameters */
 static int mca_btl_ofi_component_register(void)
 {
+    int ret;
     char *msg;
     mca_btl_ofi_module_t *module = &mca_btl_ofi_module_template;
 
@@ -191,27 +192,30 @@ static int mca_btl_ofi_component_register(void)
     /* for now we want this component to lose to the MTL. */
     module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 50;
 
-    opal_common_ofi_register_mca_variables(&mca_btl_ofi_component.super.btl_version);
+    ret = opal_common_ofi_mca_register(&mca_btl_ofi_component.super.btl_version);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
 
     return mca_btl_base_param_register(&mca_btl_ofi_component.super.btl_version, &module->super);
 }
 
 static int mca_btl_ofi_component_open(void)
 {
     mca_btl_ofi_component.module_count = 0;
-    return opal_common_ofi_init();
+    return opal_common_ofi_open();
 }
 
 /*
  * component cleanup - sanity checking of queue lengths
  */
 static int mca_btl_ofi_component_close(void)
 {
-    opal_common_ofi_mca_deregister();
-    opal_common_ofi_fini();
+    int ret;
+    ret = opal_common_ofi_close();
     /* If we don't sleep, sockets provider freaks out. Ummm this is a scary comment */
     sleep(1);
-    return OPAL_SUCCESS;
+    return ret;
 }
 
 void mca_btl_ofi_exit(void)
@@ -259,8 +263,6 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
     struct fi_domain_attr domain_attr = {0};
     uint64_t required_caps;
 
-    opal_common_ofi_mca_register();
-
     switch (mca_btl_ofi_component.mode) {
 
     case MCA_BTL_OFI_MODE_TWO_SIDED:

opal/mca/common/ofi/common_ofi.c

@@ -37,12 +37,9 @@
 
 opal_common_ofi_module_t opal_common_ofi = {.prov_include = NULL,
                                             .prov_exclude = NULL,
-                                            .registered = 0,
-                                            .verbose = 0};
-
+                                            .output = -1};
 static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream,usnic";
 static opal_mutex_t opal_common_ofi_mutex = OPAL_MUTEX_STATIC_INIT;
-static bool opal_common_ofi_initialized = false;
 static int opal_common_ofi_init_ref_cnt = 0;
 
 #ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
@@ -99,12 +96,11 @@ static void opal_common_ofi_mem_release_cb(void *buf, size_t length,
 
 #endif /* HAVE_STRUCT_FI_OPS_MEM_MONITOR */
 
-int opal_common_ofi_init(void)
+int opal_common_ofi_open(void)
 {
     int ret;
 
-    opal_common_ofi_init_ref_cnt++;
-    if (opal_common_ofi_initialized) {
+    if ((opal_common_ofi_init_ref_cnt++) > 0) {
         return OPAL_SUCCESS;
     }
 #ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
@@ -146,7 +142,6 @@ int opal_common_ofi_init(void)
         goto err;
     }
     opal_mem_hooks_register_release(opal_common_ofi_mem_release_cb, NULL);
-    opal_common_ofi_initialized = true;
 
     return OPAL_SUCCESS;
 err:
@@ -157,23 +152,35 @@ int opal_common_ofi_init(void)
         free(opal_common_ofi_monitor);
     }
 
+    opal_common_ofi_init_ref_cnt--;
+
     return OPAL_ERROR;
 #else
-    opal_common_ofi_initialized = true;
     return OPAL_SUCCESS;
 #endif
 }
 
-int opal_common_ofi_fini(void)
+int opal_common_ofi_close(void)
 {
-    if (opal_common_ofi_initialized && !--opal_common_ofi_init_ref_cnt) {
-#if OPAL_OFI_IMPORT_MONITOR_SUPPORT
-        opal_mem_hooks_unregister_release(opal_common_ofi_mem_release_cb);
-        fi_close(opal_common_ofi_cache_fid);
-        fi_close(&opal_common_ofi_monitor->fid);
-        free(opal_common_ofi_monitor);
+    int ret;
+
+    if ((--opal_common_ofi_init_ref_cnt) > 0) {
+        return OPAL_SUCCESS;
+    }
+
+#ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
+    opal_mem_hooks_unregister_release(opal_common_ofi_mem_release_cb);
+    fi_close(opal_common_ofi_cache_fid);
+    fi_close(&opal_common_ofi_monitor->fid);
+    free(opal_common_ofi_monitor);
 #endif
-        opal_common_ofi_initialized = false;
+
+    if (opal_common_ofi.output != -1) {
+        opal_output_close(opal_common_ofi.output);
+        opal_common_ofi.output = -1;
+        if (OPAL_SUCCESS != ret) {
+            return ret;
+        }
     }
 
     return OPAL_SUCCESS;
@@ -198,11 +205,12 @@ int opal_common_ofi_is_in_list(char **list, char *item)
     return 0;
 }
 
-int opal_common_ofi_register_mca_variables(const mca_base_component_t *component)
+int opal_common_ofi_mca_register(const mca_base_component_t *component)
 {
     static int include_index;
     static int exclude_index;
     static int verbose_index;
+    int verbose;
     int param;
 
     if (fi_version() < FI_VERSION(1, 0)) {
@@ -260,7 +268,7 @@ int opal_common_ofi_register_mca_variables(const mca_base_component_t *component
                                               MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                               MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                               MCA_BASE_VAR_SCOPE_LOCAL,
-                                              &opal_common_ofi.verbose);
+                                              &verbose);
     } else {
         verbose_index = param;
     }
@@ -277,32 +285,14 @@ int opal_common_ofi_register_mca_variables(const mca_base_component_t *component
                                       "verbose", 0);
     }
 
-    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
-
-    return OPAL_SUCCESS;
-}
-
-void opal_common_ofi_mca_register(void)
-{
-    opal_common_ofi.registered++;
-    if (opal_common_ofi.registered > 1) {
-        opal_output_set_verbosity(opal_common_ofi.output, opal_common_ofi.verbose);
-        return;
+    if (opal_common_ofi.output == -1) {
+        opal_common_ofi.output = opal_output_open(NULL);
+        opal_output_set_verbosity(opal_common_ofi.output, verbose);
     }
 
-    opal_common_ofi.output = opal_output_open(NULL);
-    opal_output_set_verbosity(opal_common_ofi.output, opal_common_ofi.verbose);
-}
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
 
-void opal_common_ofi_mca_deregister(void)
-{
-    /* unregister only on last deregister */
-    opal_common_ofi.registered--;
-    assert(opal_common_ofi.registered >= 0);
-    if (opal_common_ofi.registered) {
-        return;
-    }
-    opal_output_close(opal_common_ofi.output);
+    return OPAL_SUCCESS;
 }
 
 /* check that the tx attributes match */

opal/mca/common/ofi/common_ofi.h

@@ -26,46 +26,46 @@ BEGIN_C_DECLS
 typedef struct opal_common_ofi_module {
     char **prov_include;
     char **prov_exclude;
-    int verbose;
-    int registered;
     int output;
 } opal_common_ofi_module_t;
 
 extern opal_common_ofi_module_t opal_common_ofi;
 
-
 /**
- * Register component-specialized MCA variables
+ * Common MCA registration
  *
- * Register MCA variables common to all OFI components on behalf of
- * the calling component.  Expected to be called during
- * component_register for all OFI-related components.
+ * Common MCA registration handlinge.  After calling this function,
+ * \code opal_common_ofi.output will be properly initialized.
  *
  * @param component (IN) OFI component being initialized
  *
  * @returns OPAL_SUCCESS on success, OPAL error code on failure
  */
-OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_component_t *component);
+OPAL_DECLSPEC int opal_common_ofi_mca_register(const mca_base_component_t *component);
 
 /**
- * Common MCA registration
+ * Initializes common objects for libfabric
  *
- * Common MCA registration handlinge.  After calling this function,
- * \code opal_common_ofi.output will be properly initialized.
+ * Initialize common libfabric interface.  This should be called from
+ * any other OFI component's component_open() call.
  *
- * @returns OPAL_SUCCESS on success, OPAL error code on failure
+ * @note This function is not thread safe and must be called in a
+ * serial portion of the code.
  */
-OPAL_DECLSPEC void opal_common_ofi_mca_register(void);
+OPAL_DECLSPEC int opal_common_ofi_open(void);
 
 /**
- * Common MCA cleanup
+ * Cleans up common objects for libfabric
  *
- * Cleanup for any resources registered during \code
- * opal_common_ofi_mca_register().
+ * Clean up common libfabric interface.  This should be called from
+ * any other OFI component's component_close() call.  Resource cleanup
+ * is reference counted, so any successful call to
+ * opal_common_ofi_init().
  *
- * @returns OPAL_SUCCESS on success, OPAL error code on failure
+ * @note This function is not thread safe and must be called in a
+ * serial portion of the code.
  */
-OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void);
+OPAL_DECLSPEC int opal_common_ofi_close(void);
 
 /**
  * Search function for provider names
@@ -85,22 +85,6 @@ OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void);
  */
 OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item);
 
-/**
- * Initializes common objects for libfabric
- *
- * @note This function is not thread safe and must be called in a
- * serial portion of the code.
- */
-OPAL_DECLSPEC int opal_common_ofi_init(void);
-
-/**
- * Cleans up common objects for libfabric
- *
- * @note This function is not thread safe and must be called in a
- * serial portion of the code.
- */
-OPAL_DECLSPEC int opal_common_ofi_fini(void);
-
 /**
  * Selects NIC (provider) based on hardware locality
  *