Skip to content

Topic/pr7201 to v40x #7356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 12, 2020
Merged

Topic/pr7201 to v40x #7356

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8256799
hwloc/base: fix opal proc locality wrt to NUMA nodes on hwloc 2.0
bgoglin Nov 27, 2019
bed0ce7
fix a problem with opal_asprintf
hppritcha Feb 3, 2020
6702a4f
opal/hwloc: remove some unused variables when building with hwloc < 1.7
bgoglin Feb 4, 2020
f136804
hwloc/base: fix opal proc locality wrt to NUMA nodes on hwloc 1.11
bgoglin Feb 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 138 additions & 85 deletions opal/mca/hwloc/base/hwloc_base_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
* Copyright (C) 2018 Mellanox Technologies, Ltd.
* All rights reserved.
* Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved.
* Copyright (c) 2019 IBM Corporation. All rights reserved.
* Copyright (c) 2019 IBM Corporation. All rights reserved.
* Copyright (c) 2019-2020 Inria. All rights reserved.
* Copyright (c) 2020 Triad National Security, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -58,6 +60,7 @@

#include "opal/mca/hwloc/hwloc-internal.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/util/printf.h"

static bool topo_in_shmem = false;

Expand Down Expand Up @@ -1215,16 +1218,84 @@ int opal_hwloc_base_cpu_list_parse(const char *slot_str,
return OPAL_SUCCESS;
}

static void opal_hwloc_base_get_relative_locality_by_depth(hwloc_topology_t topo, unsigned d,
hwloc_cpuset_t loc1, hwloc_cpuset_t loc2,
opal_hwloc_locality_t *locality, bool *shared)
{
unsigned width, w;
hwloc_obj_t obj;
int sect1, sect2;

/* get the width of the topology at this depth */
width = hwloc_get_nbobjs_by_depth(topo, d);

/* scan all objects at this depth to see if
* our locations overlap with them
*/
for (w=0; w < width; w++) {
/* get the object at this depth/index */
obj = hwloc_get_obj_by_depth(topo, d, w);
/* see if our locations intersect with the cpuset for this obj */
sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
/* if both intersect, then we share this level */
if (sect1 && sect2) {
*shared = true;
switch(obj->type) {
case HWLOC_OBJ_NODE:
*locality |= OPAL_PROC_ON_NUMA;
break;
case HWLOC_OBJ_SOCKET:
*locality |= OPAL_PROC_ON_SOCKET;
break;
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == obj->attr->cache.depth) {
*locality |= OPAL_PROC_ON_L3CACHE;
} else if (2 == obj->attr->cache.depth) {
*locality |= OPAL_PROC_ON_L2CACHE;
} else {
*locality |= OPAL_PROC_ON_L1CACHE;
}
break;
#else
case HWLOC_OBJ_L3CACHE:
*locality |= OPAL_PROC_ON_L3CACHE;
break;
case HWLOC_OBJ_L2CACHE:
*locality |= OPAL_PROC_ON_L2CACHE;
break;
case HWLOC_OBJ_L1CACHE:
*locality |= OPAL_PROC_ON_L1CACHE;
break;
#endif
case HWLOC_OBJ_CORE:
*locality |= OPAL_PROC_ON_CORE;
break;
case HWLOC_OBJ_PU:
*locality |= OPAL_PROC_ON_HWTHREAD;
break;
default:
/* just ignore it */
break;
}
break;
}
/* otherwise, we don't share this
* object - but we still might share another object
* on this level, so we have to keep searching
*/
}
}

opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
char *cpuset1, char *cpuset2)
{
opal_hwloc_locality_t locality;
hwloc_obj_t obj;
unsigned depth, d, width, w;
hwloc_cpuset_t loc1, loc2;
unsigned depth, d;
bool shared;
hwloc_obj_type_t type;
int sect1, sect2;
hwloc_cpuset_t loc1, loc2;

/* start with what we know - they share a node on a cluster
* NOTE: we may alter that latter part as hwloc's ability to
Expand Down Expand Up @@ -1265,66 +1336,8 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
HWLOC_OBJ_PU != type) {
continue;
}
/* get the width of the topology at this depth */
width = hwloc_get_nbobjs_by_depth(topo, d);
opal_hwloc_base_get_relative_locality_by_depth(topo, d, loc1, loc2, &locality, &shared);

/* scan all objects at this depth to see if
* our locations overlap with them
*/
for (w=0; w < width; w++) {
/* get the object at this depth/index */
obj = hwloc_get_obj_by_depth(topo, d, w);
/* see if our locations intersect with the cpuset for this obj */
sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
/* if both intersect, then we share this level */
if (sect1 && sect2) {
shared = true;
switch(obj->type) {
case HWLOC_OBJ_NODE:
locality |= OPAL_PROC_ON_NUMA;
break;
case HWLOC_OBJ_SOCKET:
locality |= OPAL_PROC_ON_SOCKET;
break;
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == obj->attr->cache.depth) {
locality |= OPAL_PROC_ON_L3CACHE;
} else if (2 == obj->attr->cache.depth) {
locality |= OPAL_PROC_ON_L2CACHE;
} else {
locality |= OPAL_PROC_ON_L1CACHE;
}
break;
#else
case HWLOC_OBJ_L3CACHE:
locality |= OPAL_PROC_ON_L3CACHE;
break;
case HWLOC_OBJ_L2CACHE:
locality |= OPAL_PROC_ON_L2CACHE;
break;
case HWLOC_OBJ_L1CACHE:
locality |= OPAL_PROC_ON_L1CACHE;
break;
#endif
case HWLOC_OBJ_CORE:
locality |= OPAL_PROC_ON_CORE;
break;
case HWLOC_OBJ_PU:
locality |= OPAL_PROC_ON_HWTHREAD;
break;
default:
/* just ignore it */
break;
}
break;
}
/* otherwise, we don't share this
* object - but we still might share another object
* on this level, so we have to keep searching
*/
}
/* if we spanned the entire width without finding
* a point of intersection, then no need to go
* deeper
Expand All @@ -1333,6 +1346,9 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
break;
}
}
#if HWLOC_API_VERSION >= 0x20000
opal_hwloc_base_get_relative_locality_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, loc1, loc2, &locality, &shared);
#endif

opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
"locality: %s",
Expand All @@ -1349,9 +1365,10 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
*/
char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo)
{
#if HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC
hwloc_obj_t osdev;
unsigned i;
char **cps = NULL;
#endif
char *cpstring = NULL;
int depth;

Expand All @@ -1369,6 +1386,7 @@ char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo)
while (NULL != osdev) {
if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type) {
/* got one! find and save its serial number */
unsigned i;
for (i=0; i < osdev->infos_count; i++) {
if (0 == strncmp(osdev->infos[i].name, "MICSerialNumber", strlen("MICSerialNumber"))) {
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
Expand Down Expand Up @@ -2063,12 +2081,40 @@ char* opal_hwloc_base_get_topo_signature(hwloc_topology_t topo)
return sig;
}

static int opal_hwloc_base_get_locality_string_by_depth(hwloc_topology_t topo,
int d,
hwloc_cpuset_t cpuset,
hwloc_cpuset_t result)
{
hwloc_obj_t obj;
unsigned width, w;

/* get the width of the topology at this depth */
width = hwloc_get_nbobjs_by_depth(topo, d);
if (0 == width) {
return -1;
}

/* scan all objects at this depth to see if
* the location overlaps with them
*/
for (w=0; w < width; w++) {
/* get the object at this depth/index */
obj = hwloc_get_obj_by_depth(topo, d, w);
/* see if the location intersects with it */
if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
hwloc_bitmap_set(result, w);
}
}

return 0;
}

char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
char *bitmap)
{
hwloc_obj_t obj;
char *locality=NULL, *tmp, *t2;
unsigned depth, d, width, w;
unsigned depth, d;
hwloc_cpuset_t cpuset, result;
hwloc_obj_type_t type;

Expand Down Expand Up @@ -2111,28 +2157,15 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
continue;
}

/* get the width of the topology at this depth */
width = hwloc_get_nbobjs_by_depth(topo, d);
if (0 == width) {
if (opal_hwloc_base_get_locality_string_by_depth(topo, d, cpuset, result) < 0) {
continue;
}

/* scan all objects at this depth to see if
* the location overlaps with them
*/
for (w=0; w < width; w++) {
/* get the object at this depth/index */
obj = hwloc_get_obj_by_depth(topo, d, w);
/* see if the location intersects with it */
if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
hwloc_bitmap_set(result, w);
}
}
/* it should be impossible, but allow for the possibility
* that we came up empty at this depth */
if (!hwloc_bitmap_iszero(result)) {
hwloc_bitmap_list_asprintf(&tmp, result);
switch(obj->type) {
switch(type) {
case HWLOC_OBJ_NODE:
asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
if (NULL != locality) {
Expand All @@ -2148,15 +2181,16 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
locality = t2;
break;
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == obj->attr->cache.depth) {
case HWLOC_OBJ_CACHE: {
unsigned cachedepth = hwloc_get_obj_by_depth(topo, d, 0)->attr->cache.depth;
if (3 == cachedepth) {
asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
if (NULL != locality) {
free(locality);
}
locality = t2;
break;
} else if (2 == obj->attr->cache.depth) {
} else if (2 == cachedepth) {
asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
if (NULL != locality) {
free(locality);
Expand All @@ -2172,6 +2206,7 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
break;
}
break;
}
#else
case HWLOC_OBJ_L3CACHE:
asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
Expand Down Expand Up @@ -2217,6 +2252,24 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
}
hwloc_bitmap_zero(result);
}

#if HWLOC_API_VERSION >= 0x20000
if (opal_hwloc_base_get_locality_string_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, cpuset, result) == 0) {
/* it should be impossible, but allow for the possibility
* that we came up empty at this depth */
if (!hwloc_bitmap_iszero(result)) {
hwloc_bitmap_list_asprintf(&tmp, result);
asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
if (NULL != locality) {
free(locality);
}
locality = t2;
free(tmp);
}
hwloc_bitmap_zero(result);
}
#endif

hwloc_bitmap_free(result);
hwloc_bitmap_free(cpuset);

Expand Down