Merge pull request #3302 from ggouaillardet/poc/hwloc2

POC: add support for hwloc v2

Author: Ralph Castain

ompi/mca/topo/treematch/treematch/tm_hwloc.c

@@ -159,7 +159,11 @@ tm_topology_t* hwloc_to_tm(char *filename,double **pcost)
     exit(-1);
   }
 
+#if HWLOC_API_VERSION < 0x20000
   hwloc_topology_ignore_all_keep_structure(topology);
+#else
+#warning FIXME hwloc v2
+#endif
   hwloc_topology_load(topology);
 
 
@@ -229,7 +233,11 @@ tm_topology_t* get_local_topo_with_hwloc(void)
 
   /* Build the topology */
   hwloc_topology_init(&topology);
+#if HWLOC_API_VERSION < 0x20000
   hwloc_topology_ignore_all_keep_structure(topology);
+#else
+#warning FIXME hwloc v2
+#endif
   hwloc_topology_load(topology);
 
   /* Test if symetric */

ompi/mpiext/affinity/c/mpiext_affinity_str.c

@@ -131,7 +131,7 @@ static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
 
     /* get our root object */
     root = hwloc_get_root_obj(opal_hwloc_topology);
-    rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
+    rootset = root->cpuset;
 
     /* get our bindings */
     boundset = hwloc_bitmap_alloc();
@@ -324,7 +324,7 @@ static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
 
     /* get our root object */
     root = hwloc_get_root_obj(opal_hwloc_topology);
-    rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
+    rootset = root->cpuset;
 
     /* get our bindings */
     boundset = hwloc_bitmap_alloc();

opal/mca/btl/openib/btl_openib_component.c

@@ -19,7 +19,7 @@
  * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
  * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
  * $COPYRIGHT$
@@ -2331,6 +2331,7 @@ static float get_ib_dev_distance(struct ibv_device *dev)
        because we have no way of measuring. */
     float distance = 0;
 
+#if HWLOC_API_VERSION < 0x20000
     /* Override any distance logic so all devices are used */
     if (0 != mca_btl_openib_component.ignore_locality ||
         OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
@@ -2475,6 +2476,9 @@ static float get_ib_dev_distance(struct ibv_device *dev)
     if (NULL != my_cpuset) {
         hwloc_bitmap_free(my_cpuset);
     }
+#else
+#warning FIXME get_ib_dev_distance is not implemented with hwloc v2
+#endif
 
     return distance;
 }

opal/mca/btl/smcuda/btl_smcuda.c

@@ -16,7 +16,7 @@
  *                         reserved.
  * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -296,7 +296,6 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
             num_mem_nodes > 0 && NULL != opal_process_info.cpuset) {
             int numa=0, w;
             unsigned n_bound=0;
-            hwloc_cpuset_t avail;
             hwloc_obj_t obj;
 
             /* count the number of NUMA nodes to which we are bound */
@@ -306,10 +305,8 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
                                                                    OPAL_HWLOC_AVAILABLE))) {
                     continue;
                 }
-                /* get that NUMA node's available cpus */
-                avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
-                /* see if we intersect */
-                if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) {
+                /* see if we intersect with that NUMA node's cpus */
+                if (hwloc_bitmap_intersects(obj->cpuset, opal_hwloc_my_cpuset)) {
                     n_bound++;
                     numa = w;
                 }

opal/mca/hwloc/base/base.h

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,6 +19,12 @@
 
 #include "opal/mca/hwloc/hwloc-internal.h"
 
+#if HWLOC_API_VERSION < 0x20000
+#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
+#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
+#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
+#endif
+
 /*
  * Global functions for MCA overall hwloc open and close
  */
@@ -81,6 +89,20 @@ OPAL_DECLSPEC extern char *opal_hwloc_base_topo_file;
         hwloc_bitmap_free(bind);                                        \
     } while(0);
 
+#if HWLOC_API_VERSION < 0x20000
+#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level)              \
+    do {                                                                \
+        obj = HWLOC_OBJ_CACHE;                                          \
+        cache_level = level;                                            \
+    } while(0)
+#else
+#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level)              \
+    do {                                                                \
+        obj = HWLOC_OBJ_L##level##CACHE;                                \
+        cache_level = 0;                                                \
+    } while(0)
+#endif
+
 OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
                                                                           char *cpuset1, char *cpuset2);
 
@@ -132,9 +154,6 @@ typedef enum {
  */
 OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
 
-/* some critical helper functions */
-OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo);
-
 /**
  * Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
  * hwloc_topology_load()).
@@ -150,8 +169,6 @@ OPAL_DECLSPEC int opal_hwloc_base_set_topology(char *topofile);
  * Free the hwloc topology.
  */
 OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
-OPAL_DECLSPEC hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo,
-                                                                hwloc_obj_t obj);
 OPAL_DECLSPEC unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
                                                               hwloc_obj_type_t target,
                                                               unsigned cache_level,
@@ -285,6 +302,9 @@ OPAL_DECLSPEC char* opal_hwloc_base_get_location(char *locality,
 
 OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc2);
 
+OPAL_DECLSPEC int opal_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen);
+
+OPAL_DECLSPEC int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io);
 END_C_DECLS
 
 #endif /* OPAL_HWLOC_BASE_H */

opal/mca/hwloc/base/hwloc_base_dt.c

@@ -31,7 +31,7 @@ int opal_hwloc_pack(opal_buffer_t *buffer, const void *src,
         t = tarray[i];
 
         /* extract an xml-buffer representation of the tree */
-        if (0 != hwloc_topology_export_xmlbuffer(t, &xmlbuffer, &len)) {
+        if (0 != opal_hwloc_base_topology_export_xmlbuffer(t, &xmlbuffer, &len)) {
             return OPAL_ERROR;
         }
 
@@ -106,9 +106,7 @@ int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
         /* since we are loading this from an external source, we have to
          * explicitly set a flag so hwloc sets things up correctly
          */
-        if (0 != hwloc_topology_set_flags(t, (HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM |
-                                              HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM |
-                                              HWLOC_TOPOLOGY_FLAG_IO_DEVICES))) {
+        if (0 != opal_hwloc_base_topology_set_flags(t, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM, true)) {
             rc = OPAL_ERROR;
             hwloc_topology_destroy(t);
             goto cleanup;
@@ -137,11 +135,6 @@ int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
             goto cleanup;
         }
 
-        /* filter the cpus thru any default cpu set */
-        if (OPAL_SUCCESS != (rc = opal_hwloc_base_filter_cpus(t))) {
-            goto cleanup;
-        }
-
         /* pass it back */
         tarray[i] = t;
 
@@ -197,10 +190,10 @@ int opal_hwloc_compare(const hwloc_topology_t topo1,
      * where we really need to do a tree-wise search so we only compare
      * the things we care about, and ignore stuff like MAC addresses
      */
-    if (0 != hwloc_topology_export_xmlbuffer(t1, &x1, &l1)) {
+    if (0 != opal_hwloc_base_topology_export_xmlbuffer(t1, &x1, &l1)) {
         return OPAL_EQUAL;
     }
-    if (0 != hwloc_topology_export_xmlbuffer(t2, &x2, &l2)) {
+    if (0 != opal_hwloc_base_topology_export_xmlbuffer(t2, &x2, &l2)) {
         free(x1);
         return OPAL_EQUAL;
     }
@@ -269,18 +262,6 @@ static void print_hwloc_obj(char **output, char *prefix,
         free(tmp);
         tmp = tmp2;
     }
-    if (NULL != obj->online_cpuset) {
-        hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset);
-        asprintf(&tmp2, "%s%sOnline:  %s", tmp, pfx, string);
-        free(tmp);
-        tmp = tmp2;
-    }
-    if (NULL != obj->allowed_cpuset) {
-        hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset);
-        asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string);
-        free(tmp);
-        tmp = tmp2;
-    }
     if (HWLOC_OBJ_MACHINE == obj->type) {
         /* root level object - add support values */
         support = (struct hwloc_topology_support*)hwloc_topology_get_support(topo);

opal/mca/hwloc/base/hwloc_base_frame.c

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -50,9 +50,9 @@ hwloc_obj_type_t opal_hwloc_levels[] = {
     HWLOC_OBJ_MACHINE,
     HWLOC_OBJ_NODE,
     HWLOC_OBJ_SOCKET,
-    HWLOC_OBJ_CACHE,
-    HWLOC_OBJ_CACHE,
-    HWLOC_OBJ_CACHE,
+    HWLOC_OBJ_L3CACHE,
+    HWLOC_OBJ_L2CACHE,
+    HWLOC_OBJ_L1CACHE,
     HWLOC_OBJ_CORE,
     HWLOC_OBJ_PU
 };