Merge pull request #11641 from rhc54/topic/ofi

Do not compute distances when unbound

Author: lrbison

3rd-party/openpmix

@@ -5,7 +5,7 @@
  * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
  * Copyright (c) 2021      Amazon.com, Inc. or its affiliates. All rights
  *                         reserved.
  * Copyright (c) 2023      UT-Battelle, LLC.  All rights reserved.
@@ -469,35 +469,42 @@ static int check_provider_attr(struct fi_info *provider_info, struct fi_info *pr
 static int compute_dev_distances(pmix_device_distance_t **distances,
                                   size_t *ndist)
 {
-    int ret = 0;
+    int ret = OPAL_SUCCESS;
     size_t ninfo;
     pmix_info_t *info;
     pmix_cpuset_t cpuset;
-    pmix_topology_t *pmix_topo;
+    pmix_topology_t pmix_topo = PMIX_TOPOLOGY_STATIC_INIT;
     pmix_device_type_t type = PMIX_DEVTYPE_OPENFABRICS |
       PMIX_DEVTYPE_NETWORK;
 
     PMIX_CPUSET_CONSTRUCT(&cpuset);
     ret = PMIx_Get_cpuset(&cpuset, PMIX_CPUBIND_THREAD);
     if (PMIX_SUCCESS != ret) {
+        /* we are not bound */
+        ret = OPAL_ERR_NOT_BOUND;
         goto out;
     }
+    /* if we are not bound, then we cannot compute distances */
+    if (hwloc_bitmap_iszero(cpuset.bitmap) ||
+        hwloc_bitmap_isfull(cpuset.bitmap)) {
+        return OPAL_ERR_NOT_BOUND;
+    }
 
-    /* load the PMIX topology */
-    PMIx_Topology_free(pmix_topo, 1);
-    ret = PMIx_Load_topology(pmix_topo);
+    /* load the PMIX topology - this just loads a pointer to
+     * the local topology held in PMIx, so you must not
+     * free it */
+    ret = PMIx_Load_topology(&pmix_topo);
     if (PMIX_SUCCESS != ret) {
         goto out;
     }
 
     ninfo = 1;
     info = PMIx_Info_create(ninfo);
     PMIx_Info_load(&info[0], PMIX_DEVICE_TYPE, &type, PMIX_DEVTYPE);
-    ret = PMIx_Compute_distances(pmix_topo, &cpuset, info, ninfo, distances,
+    ret = PMIx_Compute_distances(&pmix_topo, &cpuset, info, ninfo, distances,
                                  ndist);
     PMIx_Info_free(info, ninfo);
 
-    PMIx_Topology_free(pmix_topo, 1);
 out:
     return ret;
 }
@@ -533,8 +540,9 @@ get_nearest_nics(int *num_distances, pmix_value_t **valin)
     PMIx_Info_destruct(&directive);
     if (ret != PMIX_SUCCESS || !val) {
         ret = compute_dev_distances(&distances, &ndist);
-        if (ret)
+        if (ret) {
             goto out;
+        }
         goto find_nearest;
     }
 
@@ -554,8 +562,9 @@ get_nearest_nics(int *num_distances, pmix_value_t **valin)
 
 find_nearest:
     nearest = calloc(sizeof(*distances), ndist);
-    if (!nearest)
+    if (!nearest) {
         goto out;
+    }
 
     for (i = 0; i < ndist; i++) {
         if (distances[i].type != PMIX_DEVTYPE_NETWORK &&
@@ -596,6 +605,15 @@ get_nearest_nics(int *num_distances, pmix_value_t **valin)
  *           distances array is not provided. False otherwise.
  *
  */
+#if HWLOC_API_VERSION < 0x00020000
+static bool is_near(pmix_device_distance_t *distances,
+                    int num_distances,
+                    hwloc_topology_t topology,
+                    struct fi_pci_attr pci)
+{
+    return true;
+}
+#else
 static bool is_near(pmix_device_distance_t *distances,
                     int num_distances,
                     hwloc_topology_t topology,
@@ -658,6 +676,7 @@ static bool is_near(pmix_device_distance_t *distances,
     return false;
 }
 #endif
+#endif  // OPAL_OFI_PCI_DATA_AVAILABLE
 
 /* Count providers returns the number of providers present in an fi_info list
  *     @param (IN) provider_list    struct fi_info* list of providers available
@@ -772,8 +791,8 @@ struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
     pmix_value_t *pmix_val;
     struct fi_pci_attr pci;
     int num_distances = 0;
-    bool near;
 #endif
+    bool near;
     int ret;
     unsigned int num_provider = 0, provider_limit = 0;
     bool provider_found = false;