Skip to content

Commit 6bc1d51

Browse files
edoapraBruce J Palmer
edoapra
and
Bruce J Palmer
authored
Modified PR runtime to use hostnames instead of host IDs to keep track of SMP (#357)
nodes. Co-authored-by: Bruce J Palmer <d3g293@deception04.pnl.gov>
1 parent b2c538e commit 6bc1d51

File tree

3 files changed

+58
-31
lines changed

3 files changed

+58
-31
lines changed

‎comex/src-mpi-pr/comex.c

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2722,8 +2722,8 @@ int comex_malloc_mem_dev(void *ptrs[], size_t size, comex_group_t group,
27222722
reg_entries_local[reg_entries_local_count++] = reg_entries[i];
27232723
}
27242724
}
2725-
else if (g_state.hostid[reg_entries[i].rank]
2726-
== g_state.hostid[my_world_rank]) {
2725+
else if (!strcmp(g_state.host[reg_entries[i].rank].name,
2726+
g_state.host[my_world_rank].name)) {
27272727
/* same SMP node, need to mmap */
27282728
/* open remote shared memory object */
27292729
void *memory = _shm_attach_memdev(reg_entries[i].name,
@@ -2880,7 +2880,7 @@ void _malloc_semaphore()
28802880
if (g_state.rank == i) {
28812881
continue; /* skip my own rank */
28822882
}
2883-
else if (g_state.hostid[g_state.rank] == g_state.hostid[i]) {
2883+
else if (!strcmp(g_state.host[g_state.rank].name,g_state.host[i].name)) {
28842884
/* same SMP node */
28852885
#if ENABLE_UNNAMED_SEM
28862886
semaphores[i] = _shm_attach(
@@ -2949,7 +2949,7 @@ void _free_semaphore()
29492949
}
29502950
#endif
29512951
}
2952-
else if (g_state.hostid[g_state.rank] == g_state.hostid[i]) {
2952+
else if (!strcmp(g_state.host[g_state.rank].name,g_state.host[i].name)) {
29532953
/* same SMP node */
29542954
#if ENABLE_UNNAMED_SEM
29552955
retval = munmap(semaphores[i], sizeof(sem_t));
@@ -4648,8 +4648,8 @@ STATIC void _malloc_handler(
46484648
fprintf(stderr, "[%d] _malloc_handler found NULL at %d\n", g_state.rank, i);
46494649
#endif
46504650
}
4651-
else if (g_state.hostid[reg_entries[i].rank]
4652-
== g_state.hostid[g_state.rank]) {
4651+
else if (!strcmp(g_state.host[reg_entries[i].rank].name,
4652+
g_state.host[g_state.rank].name)) {
46534653
/* same SMP node, need to mmap */
46544654
/* attach to remote shared memory object */
46554655
void *memory;
@@ -4741,8 +4741,8 @@ STATIC void _free_handler(header_t *header, char *payload, int proc)
47414741
fprintf(stderr, "[%d] _free_handler found NULL at %d\n", g_state.rank, i);
47424742
#endif
47434743
}
4744-
else if (g_state.hostid[rank_ptrs[i].rank]
4745-
== g_state.hostid[g_state.rank]) {
4744+
else if (!strcmp(g_state.host[rank_ptrs[i].rank].name,
4745+
g_state.host[g_state.rank].name)) {
47464746
/* same SMP node */
47474747
reg_entry_t *reg_entry = NULL;
47484748
int retval = 0;
@@ -4926,7 +4926,7 @@ STATIC int _smallest_world_rank_with_same_hostid(comex_igroup_t *igroup)
49264926
int *world_ranks = _get_world_ranks(igroup);
49274927

49284928
for (i=0; i<igroup->size; ++i) {
4929-
if (g_state.hostid[world_ranks[i]] == g_state.hostid[g_state.rank]) {
4929+
if (!strcmp(g_state.host[world_ranks[i]].name,g_state.host[g_state.rank].name)) {
49304930
/* found same host as me */
49314931
if (world_ranks[i] < smallest) {
49324932
smallest = world_ranks[i];
@@ -4949,7 +4949,7 @@ STATIC int _largest_world_rank_with_same_hostid(comex_igroup_t *igroup)
49494949
int *world_ranks = _get_world_ranks(igroup);
49504950

49514951
for (i=0; i<igroup->size; ++i) {
4952-
if (g_state.hostid[world_ranks[i]] == g_state.hostid[g_state.rank]) {
4952+
if (!strcmp(g_state.host[world_ranks[i]].name,g_state.host[g_state.rank].name)) {
49534953
/* found same host as me */
49544954
if (world_ranks[i] > largest) {
49554955
largest = world_ranks[i];
@@ -6365,7 +6365,7 @@ STATIC void nb_puts(
63656365
if (COMEX_ENABLE_PUT_DATATYPE
63666366
&& (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc)
63676367
&& (!COMEX_ENABLE_PUT_SMP
6368-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])
6368+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))
63696369
&& (_packed_size(src_stride, count, stride_levels) > COMEX_PUT_DATATYPE_THRESHOLD)) {
63706370
nb_puts_datatype(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb);
63716371
return;
@@ -6375,7 +6375,7 @@ STATIC void nb_puts(
63756375
if (COMEX_ENABLE_PUT_PACKED
63766376
&& (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc)
63776377
&& (!COMEX_ENABLE_PUT_SMP
6378-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
6378+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
63796379
nb_puts_packed(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb);
63806380
return;
63816381
}
@@ -6634,7 +6634,7 @@ STATIC void nb_gets(
66346634
if (COMEX_ENABLE_GET_DATATYPE
66356635
&& (!COMEX_ENABLE_GET_SELF || g_state.rank != proc)
66366636
&& (!COMEX_ENABLE_GET_SMP
6637-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])
6637+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))
66386638
&& (_packed_size(src_stride, count, stride_levels) > COMEX_GET_DATATYPE_THRESHOLD)) {
66396639
nb_gets_datatype(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb);
66406640
return;
@@ -6644,7 +6644,7 @@ STATIC void nb_gets(
66446644
if (COMEX_ENABLE_GET_PACKED
66456645
&& (!COMEX_ENABLE_GET_SELF || g_state.rank != proc)
66466646
&& (!COMEX_ENABLE_GET_SMP
6647-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
6647+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
66486648
nb_gets_packed(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb);
66496649
return;
66506650
}
@@ -6910,7 +6910,7 @@ STATIC void nb_accs(
69106910
if (COMEX_ENABLE_ACC_PACKED
69116911
&& (!COMEX_ENABLE_ACC_SELF || g_state.rank != proc)
69126912
&& (!COMEX_ENABLE_ACC_SMP
6913-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
6913+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
69146914
nb_accs_packed(datatype, scale, src, src_stride, dst, dst_stride, count, stride_levels, proc, nb);
69156915
return;
69166916
}
@@ -7120,7 +7120,7 @@ STATIC void nb_putv(
71207120
if (COMEX_ENABLE_PUT_IOV
71217121
&& (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc)
71227122
&& (!COMEX_ENABLE_PUT_SMP
7123-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
7123+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
71247124
nb_putv_packed(&iov[i], proc, nb);
71257125
}
71267126
else {
@@ -7224,7 +7224,7 @@ STATIC void nb_getv(
72247224
if (COMEX_ENABLE_GET_IOV
72257225
&& (!COMEX_ENABLE_GET_SELF || g_state.rank != proc)
72267226
&& (!COMEX_ENABLE_GET_SMP
7227-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
7227+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
72287228
nb_getv_packed(&iov[i], proc, nb);
72297229
}
72307230
else {
@@ -7336,7 +7336,7 @@ STATIC void nb_accv(
73367336
if (COMEX_ENABLE_ACC_IOV
73377337
&& (!COMEX_ENABLE_ACC_SELF || g_state.rank != proc)
73387338
&& (!COMEX_ENABLE_ACC_SMP
7339-
|| g_state.hostid[proc] != g_state.hostid[g_state.rank])) {
7339+
|| strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) {
73407340
nb_accv_packed(datatype, scale, &iov[i], proc, nb);
73417341
}
73427342
else {

‎comex/src-mpi-pr/groups.c

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,27 @@ static int cmplong(const void *p1, const void *p2)
381381
return *((long*)p1) - *((long*)p2);
382382
}
383383

384+
static int cmpname(const void *name1, const void *name2)
385+
{
386+
const char* n1 = (const char*)name1;
387+
const char* n2 = (const char*)name2;
388+
int comp = 0;
389+
int i;
390+
for (i=0; i<COMEX_MAX_HOST_NAME_LEN; i++) {
391+
if ((int)n1[i] < (int)n2[i]) {
392+
comp = -1;
393+
break;
394+
} else if ((int)n1[i] > (int)n2[i]) {
395+
comp = 1;
396+
break;
397+
} else if (n1[i] == '\0' || n2[i] == '\0') {
398+
break;
399+
}
400+
}
401+
return comp;
402+
}
403+
404+
384405
/**
385406
* Initialize group linked list. Prepopulate with world group.
386407
*/
@@ -393,7 +414,7 @@ void comex_group_init(MPI_Comm comm)
393414
int size_node = 0;
394415
comex_group_t group = 0;
395416
comex_igroup_t *igroup = NULL;
396-
long *sorted = NULL;
417+
host_name_t *sorted = NULL;
397418
int count = 0;
398419

399420
/* populate g_state */
@@ -420,10 +441,10 @@ void comex_group_init(MPI_Comm comm)
420441
#endif
421442

422443
/* need to figure out which proc is master on each node */
423-
g_state.hostid = (long*)malloc(sizeof(long)*g_state.size);
424-
g_state.hostid[g_state.rank] = xgethostid();
425-
status = MPI_Allgather(MPI_IN_PLACE, 1, MPI_LONG,
426-
g_state.hostid, 1, MPI_LONG, g_state.comm);
444+
g_state.host = (host_name_t*)malloc(sizeof(host_name_t)*g_state.size);
445+
gethostname(g_state.host[g_state.rank].name,COMEX_MAX_HOST_NAME_LEN);
446+
status = MPI_Allgather(MPI_IN_PLACE, sizeof(host_name_t), MPI_BYTE,
447+
g_state.host, sizeof(host_name_t), MPI_BYTE, g_state.comm);
427448
COMEX_ASSERT(MPI_SUCCESS == status);
428449
/* First create a temporary node communicator and then
429450
* split further into number of groups within the node */
@@ -432,17 +453,17 @@ void comex_group_init(MPI_Comm comm)
432453
/* create node comm */
433454
/* MPI_Comm_split requires a non-negative color,
434455
* so sort and sanitize */
435-
sorted = (long*)malloc(sizeof(long) * g_state.size);
436-
(void)memcpy(sorted, g_state.hostid, sizeof(long)*g_state.size);
437-
qsort(sorted, g_state.size, sizeof(long), cmplong);
456+
sorted = (long*)malloc(sizeof(host_name_t) * g_state.size);
457+
(void)memcpy(sorted, g_state.host, sizeof(host_name_t)*g_state.size);
458+
qsort(sorted, g_state.size, sizeof(host_name_t), cmpname);
438459
/* count is number of distinct host IDs that are lower than
439460
* the host ID of this rank */
440461
for (i=0; i<g_state.size-1; ++i) {
441-
if (sorted[i] == g_state.hostid[g_state.rank])
462+
if (!strcmp(sorted[i].name,g_state.host[g_state.rank].name))
442463
{
443464
break;
444465
}
445-
if (sorted[i] != sorted[i+1]) {
466+
if (strcmp(sorted[i].name,sorted[i+1].name)) {
446467
count += 1;
447468
}
448469
}
@@ -463,7 +484,7 @@ void comex_group_init(MPI_Comm comm)
463484
smallest_rank_with_same_hostid = g_state.rank;
464485
largest_rank_with_same_hostid = g_state.rank;
465486
for (i=0; i<g_state.size; ++i) {
466-
if (g_state.hostid[i] == g_state.hostid[g_state.rank]) {
487+
if (!strcmp(g_state.host[i].name,g_state.host[g_state.rank].name)) {
467488
++size_node;
468489
if (i < smallest_rank_with_same_hostid) {
469490
smallest_rank_with_same_hostid = i;
@@ -605,7 +626,7 @@ void comex_group_finalize()
605626
}
606627

607628
free(g_state.master);
608-
free(g_state.hostid);
629+
free(g_state.host);
609630
status = MPI_Comm_free(&(g_state.node_comm));
610631
COMEX_ASSERT(MPI_SUCCESS == status);
611632
status = MPI_Group_free(&(g_state.group));

‎comex/src-mpi-pr/groups.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,23 @@
88
#ifndef _COMEX_GROUPS_H_
99
#define _COMEX_GROUPS_H_
1010

11+
#define COMEX_MAX_HOST_NAME_LEN 256
12+
1113
#include <mpi.h>
1214

1315
#include "comex.h"
1416

17+
typedef struct {
18+
char name[COMEX_MAX_HOST_NAME_LEN];
19+
} host_name_t;
20+
1521
typedef struct {
1622
MPI_Comm comm; /**< whole comm; all ranks */
1723
MPI_Group group;/**< whole group; all ranks */
1824
int size; /**< comm size */
1925
int rank; /**< comm rank */
2026
int *master; /**< master[size] rank of a given rank's master */
21-
long *hostid; /**< hostid[size] hostid of SMP node for a given rank */
27+
host_name_t *host; /**< host[size] host name of SMP node for a given rank */
2228
MPI_Comm node_comm; /**< node comm; SMP ranks */
2329
int node_size; /**< node comm size */
2430
int node_rank; /**< node comm rank */

0 commit comments

Comments
 (0)