Merge pull request #13319 from bwbarrett/backports/v5.0-rdma-when-mtl-fix

osc: Fix rdma component when not using ob1

Author: janjust

ompi/instance/instance.c

@@ -535,6 +535,10 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_pml_base_select() failed", ret);
     }
 
+    if (OMPI_SUCCESS != (ret = ompi_osc_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+        return ompi_instance_print_error ("ompi_osc_base_find_available() failed", ret);
+    }
+
     OMPI_TIMING_IMPORT_OPAL("orte_init");
     OMPI_TIMING_NEXT("rte_init-commit");
 
@@ -616,10 +620,6 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_coll_base_find_available() failed", ret);
     }
 
-    if (OMPI_SUCCESS != (ret = ompi_osc_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
-        return ompi_instance_print_error ("ompi_osc_base_find_available() failed", ret);
-    }
-
     /* io and topo components are not selected here -- see comment
        above about the io and topo frameworks being loaded lazily */
 
@@ -653,7 +653,8 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("ompi_attr_create_predefined_keyvals() failed", ret);
     }
 
-    if (mca_pml_base_requires_world ()) {
+    if (mca_pml_base_requires_world() ||
+        mca_osc_base_requires_world()) {
         /* need to set up comm world for this instance -- XXX -- FIXME -- probably won't always
          * be the case. */
         if (OMPI_SUCCESS != (ret = ompi_comm_init_mpi3 ())) {
@@ -702,7 +703,8 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
     /* some btls/mtls require we call add_procs with all procs in the job.
      * since the btls/mtls have no visibility here it is up to the pml to
      * convey this requirement */
-    if (mca_pml_base_requires_world ()) {
+    if (mca_pml_base_requires_world() ||
+        mca_osc_base_requires_world()) {
         if (NULL == (procs = ompi_proc_world (&nprocs))) {
             return ompi_instance_print_error ("ompi_proc_get_allocated () failed", ret);
         }

ompi/mca/osc/base/osc_base_init.c

@@ -30,6 +30,8 @@
 #include "ompi/communicator/communicator.h"
 #include "ompi/win/win.h"
 
+bool ompi_osc_base_requires_world = false;
+
 int
 ompi_osc_base_select(ompi_win_t *win,
                      void **base,

ompi/mca/osc/osc.h

@@ -53,6 +53,9 @@ struct ompi_datatype_t;
 struct ompi_op_t;
 struct ompi_request_t;
 
+
+extern bool ompi_osc_base_requires_world;
+
 /* ******************************************************************** */
 
 
@@ -419,6 +422,11 @@ typedef ompi_osc_base_module_3_0_0_t ompi_osc_base_module_t;
 
 /* ******************************************************************** */
 
+static inline bool mca_osc_base_requires_world (void)
+{
+    return ompi_osc_base_requires_world;
+}
+
 
 END_C_DECLS
 

ompi/mca/osc/portals4/osc_portals4_component.c

@@ -348,6 +348,8 @@ component_init(bool enable_progress_threads, bool enable_mpi_threads)
         return ret;
     }
 
+    ompi_osc_base_requires_world = true;
+
     return OMPI_SUCCESS;
 }
 

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -344,6 +344,27 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads,
                             __FILE__, __LINE__, ret);
     }
 
+    ret = mca_bml_base_init(enable_progress_threads, enable_mpi_threads);
+    if (OPAL_SUCCESS != ret) {
+        opal_output_verbose(1, ompi_osc_base_framework.framework_output,
+                            "%s:%d: bml_base_init() failed: %d",
+                            __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    /* check if any btls do not support dynamic add_procs */
+    mca_btl_base_selected_module_t* selected_btl;
+    OPAL_LIST_FOREACH(selected_btl, &mca_btl_base_modules_initialized,
+                      mca_btl_base_selected_module_t) {
+        mca_btl_base_module_t *btl = selected_btl->btl_module;
+
+        if (btl->btl_flags & MCA_BTL_FLAGS_SINGLE_ADD_PROCS) {
+            ompi_osc_base_requires_world = true;
+            break;
+        }
+
+    }
+
     return ret;
 }