Skip to content

Commit 567ab68

Browse files
bgoglinbgoglin
committed
linux/knl: add an envvar to expose KNL MCDRAM Cache as a hwloc MS$
That's what it is for real, but we couldn't expose that before. Don't make it the default to avoid breaking existing codes. Signed-off-by: Brice Goglin <Brice.Goglin@inria.fr>
1 parent e042415 commit 567ab68

File tree

3 files changed

+49
-19
lines changed

3 files changed

+49
-19
lines changed

‎NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ Version 2.1.0
2121
* API
2222
+ Add the new HWLOC_OBJ_MEMCACHE object type for memory-side caches.
2323
- They are filtered-out by default, except in command-line tools.
24+
- The KNL MCDRAM in cache mode is still exposed as a L3 unless
25+
HWLOC_KNL_MSCACHE_L3=0 in the environment.
2426
+ Add HWLOC_RESTRICT_FLAG_BYNODESET and _REMOVE_MEMLESS for restricting
2527
topologies based on some memory nodes.
2628
+ Add hwloc_topology_set_components() for blacklisting some components

‎doc/hwloc.doxy‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,14 @@ following environment variables.
10801080
corresponding GPU.
10811081
</dd>
10821082

1083+
<dt>HWLOC_KNL_MSCACHE_L3=0</dt>
1084+
<dd>Expose the KNL MCDRAM in cache mode as a Memory-side Cache instead of a L3.
1085+
hwloc releases prior to 2.1 exposed the MCDRAM cache as a CPU-side L3 cache.
1086+
Now that Memory-side caches are supported by hwloc, it is still exposed
1087+
as a L3 by default to avoid breaking existing applications.
1088+
Setting this environment variable to 1 will expose it as a proper Memory-side cache.
1089+
</dd>
1090+
10831091
<dt>HWLOC_FSROOT=/path/to/linux/filesystem-root/</dt>
10841092
<dd>switches to reading the topology from the specified
10851093
Linux filesystem root instead of the main file-system root.
@@ -3625,7 +3633,9 @@ $ hwloc-bind --membind numa:0 -- myprogram
36253633
\subsection faq_knl_dump Why do I need hwloc-dump-hwdata for memory on Intel Xeon Phi processor?
36263634

36273635
Intel Xeon Phi processors may use the on-package memory (MCDRAM)
3628-
as either memory or a memory-side cache (currently reported as a L3 cache by hwloc).
3636+
as either memory or a memory-side cache
3637+
(reported as a L3 cache by hwloc by default,
3638+
see <tt>HWLOC_KNL_MSCACHE_L3</tt> in \ref envvar).
36293639
There are also several clustering modes that significantly affect the memory organization
36303640
(see \ref faq_knl_numa for more information about these modes).
36313641
Details about these are currently only available to privileged users.

‎hwloc/topology-linux.c‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3481,6 +3481,7 @@ static void
34813481
hwloc_linux_knl_add_cluster(struct hwloc_topology *topology,
34823482
hwloc_obj_t ddr, hwloc_obj_t mcdram,
34833483
struct knl_hwdata *knl_hwdata,
3484+
int mscache_as_l3,
34843485
unsigned *failednodes)
34853486
{
34863487
hwloc_obj_t cluster = NULL;
@@ -3543,8 +3544,18 @@ hwloc_linux_knl_add_cluster(struct hwloc_topology *topology,
35433544
hwloc_obj_add_info(cache, "Inclusive", knl_hwdata->mcdram_cache_inclusiveness ? "1" : "0");
35443545
cache->cpuset = hwloc_bitmap_dup(ddr->cpuset);
35453546
cache->nodeset = hwloc_bitmap_dup(ddr->nodeset); /* only applies to DDR */
3546-
cache->subtype = strdup("MemorySideCache");
3547-
hwloc_insert_object_by_cpuset(topology, cache);
3547+
if (mscache_as_l3) {
3548+
/* make it a L3 */
3549+
cache->subtype = strdup("MemorySideCache");
3550+
hwloc_insert_object_by_cpuset(topology, cache);
3551+
} else {
3552+
/* make it a real mscache */
3553+
cache->type = HWLOC_OBJ_MEMCACHE;
3554+
if (cluster)
3555+
hwloc__attach_memory_object(topology, cluster, cache, hwloc_report_os_error);
3556+
else
3557+
hwloc__insert_object_by_cpuset(topology, NULL, cache, hwloc_report_os_error);
3558+
}
35483559
}
35493560
}
35503561

@@ -3560,6 +3571,8 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
35603571
unsigned i;
35613572
char * fallback_env = getenv("HWLOC_KNL_HDH_FALLBACK");
35623573
int fallback = fallback_env ? atoi(fallback_env) : -1; /* by default, only fallback if needed */
3574+
char * mscache_as_l3_env = getenv("HWLOC_KNL_MSCACHE_L3");
3575+
int mscache_as_l3 = mscache_as_l3_env ? atoi(mscache_as_l3_env) : 1; /* L3 by default, for backward compat */
35633576

35643577
if (*failednodes)
35653578
goto error;
@@ -3596,8 +3609,13 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
35963609
goto error;
35973610
}
35983611

3599-
if (!hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_L3CACHE))
3600-
hwdata.mcdram_cache_size = 0;
3612+
if (mscache_as_l3) {
3613+
if (!hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_L3CACHE))
3614+
hwdata.mcdram_cache_size = 0;
3615+
} else {
3616+
if (!hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_MEMCACHE))
3617+
hwdata.mcdram_cache_size = 0;
3618+
}
36013619

36023620
hwloc_obj_add_info(topology->levels[0][0], "ClusterMode", hwdata.cluster_mode);
36033621
hwloc_obj_add_info(topology->levels[0][0], "MemoryMode", hwdata.memory_mode);
@@ -3611,7 +3629,7 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36113629
fprintf(stderr, "Found %u NUMA nodes instead of 1 in mode %s-%s\n", nbnodes, hwdata.cluster_mode, hwdata.memory_mode);
36123630
goto error;
36133631
}
3614-
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, failednodes);
3632+
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, mscache_as_l3, failednodes);
36153633

36163634
} else {
36173635
/* Quadrant-Flat/Hybrid */
@@ -3621,7 +3639,7 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36213639
}
36223640
if (!strcmp(hwdata.memory_mode, "Flat"))
36233641
hwdata.mcdram_cache_size = 0;
3624-
hwloc_linux_knl_add_cluster(topology, nodes[0], nodes[1], &hwdata, failednodes);
3642+
hwloc_linux_knl_add_cluster(topology, nodes[0], nodes[1], &hwdata, mscache_as_l3, failednodes);
36253643
}
36263644

36273645
} else if (!strcmp(hwdata.cluster_mode, "SNC2")) {
@@ -3631,8 +3649,8 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36313649
fprintf(stderr, "Found %u NUMA nodes instead of 2 in mode %s-%s\n", nbnodes, hwdata.cluster_mode, hwdata.memory_mode);
36323650
goto error;
36333651
}
3634-
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, failednodes);
3635-
hwloc_linux_knl_add_cluster(topology, nodes[1], NULL, &hwdata, failednodes);
3652+
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, mscache_as_l3, failednodes);
3653+
hwloc_linux_knl_add_cluster(topology, nodes[1], NULL, &hwdata, mscache_as_l3, failednodes);
36363654

36373655
} else {
36383656
/* SNC2-Flat/Hybrid */
@@ -3647,8 +3665,8 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36473665
}
36483666
if (!strcmp(hwdata.memory_mode, "Flat"))
36493667
hwdata.mcdram_cache_size = 0;
3650-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[0]], nodes[mcdram[0]], &hwdata, failednodes);
3651-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[1]], nodes[mcdram[1]], &hwdata, failednodes);
3668+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[0]], nodes[mcdram[0]], &hwdata, mscache_as_l3, failednodes);
3669+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[1]], nodes[mcdram[1]], &hwdata, mscache_as_l3, failednodes);
36523670
}
36533671

36543672
} else if (!strcmp(hwdata.cluster_mode, "SNC4")) {
@@ -3658,10 +3676,10 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36583676
fprintf(stderr, "Found %u NUMA nodes instead of 4 in mode %s-%s\n", nbnodes, hwdata.cluster_mode, hwdata.memory_mode);
36593677
goto error;
36603678
}
3661-
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, failednodes);
3662-
hwloc_linux_knl_add_cluster(topology, nodes[1], NULL, &hwdata, failednodes);
3663-
hwloc_linux_knl_add_cluster(topology, nodes[2], NULL, &hwdata, failednodes);
3664-
hwloc_linux_knl_add_cluster(topology, nodes[3], NULL, &hwdata, failednodes);
3679+
hwloc_linux_knl_add_cluster(topology, nodes[0], NULL, &hwdata, mscache_as_l3, failednodes);
3680+
hwloc_linux_knl_add_cluster(topology, nodes[1], NULL, &hwdata, mscache_as_l3, failednodes);
3681+
hwloc_linux_knl_add_cluster(topology, nodes[2], NULL, &hwdata, mscache_as_l3, failednodes);
3682+
hwloc_linux_knl_add_cluster(topology, nodes[3], NULL, &hwdata, mscache_as_l3, failednodes);
36653683

36663684
} else {
36673685
/* SNC4-Flat/Hybrid */
@@ -3676,10 +3694,10 @@ hwloc_linux_knl_numa_quirk(struct hwloc_topology *topology,
36763694
}
36773695
if (!strcmp(hwdata.memory_mode, "Flat"))
36783696
hwdata.mcdram_cache_size = 0;
3679-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[0]], nodes[mcdram[0]], &hwdata, failednodes);
3680-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[1]], nodes[mcdram[1]], &hwdata, failednodes);
3681-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[2]], nodes[mcdram[2]], &hwdata, failednodes);
3682-
hwloc_linux_knl_add_cluster(topology, nodes[ddr[3]], nodes[mcdram[3]], &hwdata, failednodes);
3697+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[0]], nodes[mcdram[0]], &hwdata, mscache_as_l3, failednodes);
3698+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[1]], nodes[mcdram[1]], &hwdata, mscache_as_l3, failednodes);
3699+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[2]], nodes[mcdram[2]], &hwdata, mscache_as_l3, failednodes);
3700+
hwloc_linux_knl_add_cluster(topology, nodes[ddr[3]], nodes[mcdram[3]], &hwdata, mscache_as_l3, failednodes);
36833701
}
36843702
}
36853703

0 commit comments

Comments
 (0)