Safety valve for callbacks

When an mca_foo.so gets unloaded any callbacks it registered become invalid and can
segv if called.

This happened in an oshemem testcase that did call shmem_init() and shmem_finalize().
But for whatever reason oshmem unloads the MCAs before they call MPI_Finalize() which
they have in a conditional so they don't necessarily always call MPI_Finalize().
Anyway the failing path goes from oshmem_shmem_finalize() through _shmem_finalize()
which unloads the MCAs (mca_base_framework_close which dlcloses them) and then after
that it will call MPI_Finalize() which will deregister the callback

That could be considered a bug, maybe the reference counters should be set up so the
MCA's .fini call is always called before dlclose() and make sure all such .fini calls
deregister the MCA's callbacks.

But this checkin takes the conservative approach of assuming that bugs like that are
always a possibility and inserts a safety valve around callback usage, so when an MCA
gets unloaded it deregisters any callbacks rather than only having them deregistered
if we go through a proper sequence of finalize calls.

Signed-off-by: Austen Lauria <awlauria@us.ibm.com>

Author: markalle

ompi/mca/coll/hcoll/coll_hcoll_module.c

@@ -16,6 +16,8 @@
 #include "coll_hcoll.h"
 #include "coll_hcoll_dtypes.h"
 
+static int use_safety_valve = 0;
+
 int hcoll_comm_attr_keyval;
 int hcoll_type_attr_keyval;
 mca_coll_hcoll_dtype_t zero_dte_mapping;
@@ -327,6 +329,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
                     cm->using_mem_hooks = 1;
                     opal_mem_hooks_register_release(mca_coll_hcoll_mem_release_cb, NULL);
                     setenv("MXM_HCOLL_MEM_ON_DEMAND_MAP", "y", 0);
+                    use_safety_valve = 1;
                 }
             }
         } else {
@@ -447,5 +450,9 @@ OBJ_CLASS_INSTANCE(mca_coll_hcoll_module_t,
         mca_coll_hcoll_module_construct,
         mca_coll_hcoll_module_destruct);
 
-
-
+static void safety_valve(void) __attribute__((destructor));
+void safety_valve(void) {
+    if (use_safety_valve) {
+        opal_mem_hooks_unregister_release(mca_coll_hcoll_mem_release_cb);
+    }
+}

opal/mca/btl/uct/btl_uct_component.c

@@ -43,6 +43,8 @@
 #include "btl_uct_am.h"
 #include "btl_uct_device_context.h"
 
+static int use_safety_valve = 0;
+
 static int mca_btl_uct_component_register(void)
 {
     mca_btl_uct_module_t *module = &mca_btl_uct_module_template;
@@ -143,6 +145,7 @@ static int mca_btl_uct_component_open(void)
                 & opal_mem_hooks_support_level()))) {
         ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
         opal_mem_hooks_register_release(mca_btl_uct_mem_release_cb, NULL);
+        use_safety_valve = 1;
     }
 
     return OPAL_SUCCESS;
@@ -667,3 +670,10 @@ mca_btl_uct_component_t mca_btl_uct_component = {
         .btl_init = mca_btl_uct_component_init,
         .btl_progress = mca_btl_uct_component_progress,
     }};
+
+static void safety_valve(void) __attribute__((destructor));
+void safety_valve(void) {
+    if (use_safety_valve) {
+        opal_mem_hooks_unregister_release(mca_btl_uct_mem_release_cb);
+    }
+}

opal/mca/common/ucx/common_ucx.c

@@ -26,6 +26,8 @@
 #include <stdio.h>
 #include <ucm/api/ucm.h>
 
+static int use_safety_valve = 0;
+
 /***********************************************************************/
 
 extern mca_base_framework_t opal_memory_base_framework;
@@ -179,6 +181,7 @@ OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
             MCA_COMMON_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events");
             ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
             opal_mem_hooks_register_release(opal_common_ucx_mem_release_cb, NULL);
+            use_safety_valve = 1;
         }
     }
 }
@@ -499,3 +502,10 @@ OPAL_DECLSPEC int opal_common_ucx_del_procs(opal_common_ucx_del_proc_t *procs, s
 
     return opal_common_ucx_mca_pmix_fence(worker);
 }
+
+static void safety_valve(void) __attribute__((destructor));
+void safety_valve(void) {
+    if (use_safety_valve) {
+        opal_mem_hooks_unregister_release(opal_common_ucx_mem_release_cb);
+    }
+}

opal/mca/rcache/base/rcache_base_create.c

@@ -37,6 +37,8 @@
 #include "opal/memoryhooks/memory.h"
 #include "opal/runtime/opal_params.h"
 
+static int use_safety_valve = 0;
+
 mca_rcache_base_module_t *
 mca_rcache_base_module_create(const char *name, void *user_data,
                               struct mca_rcache_base_resources_t *resources)
@@ -70,6 +72,7 @@ mca_rcache_base_module_create(const char *name, void *user_data,
                     opal_leave_pinned = !opal_leave_pinned_pipeline;
                 }
                 opal_mem_hooks_register_release(mca_rcache_base_mem_cb, NULL);
+                use_safety_valve = 1;
             } else if (1 == opal_leave_pinned || opal_leave_pinned_pipeline) {
                 opal_show_help("help-rcache-base.txt", "leave pinned failed", true, name,
                                OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), opal_process_info.nodename);
@@ -121,3 +124,10 @@ int mca_rcache_base_module_destroy(mca_rcache_base_module_t *module)
 
     return OPAL_ERR_NOT_FOUND;
 }
+
+static void safety_valve(void) __attribute__((destructor));
+void safety_valve(void) {
+    if (use_safety_valve) {
+        opal_mem_hooks_unregister_release(mca_rcache_base_mem_cb);
+    }
+}