Merge pull request open-mpi#8470 from hjelmn/btl_vader_fix_for_dynamic_add_procs

btl/vader: improve support for dynamic add_procs

Author: hjelmn

opal/mca/btl/sm/btl_sm.h

@@ -125,7 +125,6 @@ struct mca_btl_sm_component_t {
     char *my_segment;                       /**< this rank's base pointer */
     size_t segment_size;                    /**< size of my_segment */
     int32_t num_smp_procs;                  /**< current number of smp procs on this host */
-    opal_atomic_int32_t local_rank;         /**< current rank index at add_procs() time */
     opal_free_list_t sm_frags_eager;     /**< free list of sm send frags */
     opal_free_list_t sm_frags_max_send;  /**< free list of sm max send frags (large fragments) */
     opal_free_list_t sm_frags_user;      /**< free list of small inline frags */

opal/mca/btl/sm/btl_sm_component.c

@@ -560,8 +560,6 @@ static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
     /* no fast boxes allocated initially */
     component->num_fbox_in_endpoints = 0;
 
-    component->local_rank = 0;
-
     mca_btl_sm_check_single_copy ();
 
     if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {

opal/mca/btl/sm/btl_sm_endpoint.h

@@ -64,7 +64,7 @@ typedef struct mca_btl_base_endpoint_t {
         opal_free_list_item_t *fbox; /**< fast-box free list item */
     } fbox_out;
 
-    int32_t peer_smp_rank;  /**< my peer's SMP process rank.  Used for accessing
+    uint16_t peer_smp_rank; /**< my peer's SMP process rank.  Used for accessing
                              *   SMP specfic data structures. */
     opal_atomic_size_t send_count;    /**< number of fragments sent to this peer */
     char *segment_base;     /**< start of the peer's segment (in the address space

opal/mca/btl/sm/btl_sm_module.c

@@ -194,18 +194,30 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
 }
 
 
-static int init_sm_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
+static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opal_proc_t *proc) {
     mca_btl_sm_component_t *component = &mca_btl_sm_component;
     union sm_modex_t *modex;
     ino_t my_user_ns_id;
     size_t msg_size;
     int rc;
 
+    uint16_t peer_local_rank;
+    uint16_t *ptr = &peer_local_rank;
+    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_RANK, &proc->proc_name,
+                          &ptr, PMIX_UINT16);
+    if (OPAL_SUCCESS != rc) {
+        BTL_VERBOSE(("could not read the local rank for peer. rc=%d", rc));
+        return rc;
+    }
+
+    mca_btl_base_endpoint_t *ep = component->endpoints + peer_local_rank;
+    *ep_out = ep;
+
     OBJ_CONSTRUCT(ep, mca_btl_sm_endpoint_t);
 
-    ep->peer_smp_rank = remote_rank;
+    ep->peer_smp_rank = peer_local_rank;
 
-    if (remote_rank != MCA_BTL_SM_LOCAL_RANK) {
+    if (peer_local_rank != MCA_BTL_SM_LOCAL_RANK) {
         OPAL_MODEX_RECV_IMMEDIATE(rc, &component->super.btl_version,
                                   &proc->proc_name, (void **) &modex, &msg_size);
         if (OPAL_SUCCESS != rc) {
@@ -363,10 +375,7 @@ static int sm_add_procs (struct mca_btl_base_module_t* btl,
         }
 
         /* setup endpoint */
-        int rank = opal_atomic_fetch_add_32(&component -> local_rank, 1);
-
-        peers[proc] = component->endpoints + rank;
-        rc = init_sm_endpoint (peers[proc], procs[proc], rank);
+        rc = init_sm_endpoint (peers + proc, procs[proc]);
         if (OPAL_SUCCESS != rc) {
             break;
         }