Skip to content

Commit 72b9947

Browse files
hjelmnhjelmn
committed
btl/usnic: update for mpool/rcache rewrite
Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
1 parent df5b2fc commit 72b9947

File tree

5 files changed

+85
-8
lines changed

5 files changed

+85
-8
lines changed

‎opal/mca/btl/usnic/btl_usnic.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,6 +13,8 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2011-2015 Cisco Systems, Inc. All rights reserved.
16+
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
17+
* reserved.
1518
* $COPYRIGHT$
1619
*
1720
* Additional copyrights may follow
@@ -37,7 +40,10 @@
3740
#include "opal/mca/btl/btl.h"
3841
#include "opal/mca/btl/base/btl_base_error.h"
3942
#include "opal/mca/btl/base/base.h"
43+
#include "opal/mca/rcache/rcache.h"
44+
#if RCACHE_VERSION < 30
4045
#include "opal/mca/mpool/grdma/mpool_grdma.h"
46+
#endif
4147
#else
4248
#include "ompi/mca/btl/btl.h"
4349
#include "ompi/mca/btl/base/btl_base_error.h"
@@ -153,8 +159,16 @@ typedef struct opal_btl_usnic_component_t {
153159
/** list of usnic proc structures */
154160
opal_list_t usnic_procs;
155161

162+
#if RCACHE_VERSION == 30
163+
/** memory pool hints */
164+
char* usnic_mpool_hints;
165+
166+
/** registration cache name */
167+
char *usnic_rcache_name;
168+
#else
156169
/** name of memory pool */
157170
char* usnic_mpool_name;
171+
#endif
158172

159173
char *if_include;
160174
char *if_exclude;

‎opal/mca/btl/usnic/btl_usnic_compat.h‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2013-2015 Cisco Systems, Inc. All rights reserved.
4+
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
5+
* reserved.
36
* $COPYRIGHT$
47
*
58
* Additional copyrights may follow
@@ -8,11 +11,13 @@
811
*/
912

1013
/* This header contains macros to help minimize usnic BTL differences
11-
* between v1.7/v1.8 and v1.9/v2.0. */
14+
* between v1.7/v1.8, v1.9/v2.0, and v2.0/v2.1. */
1215

1316
#ifndef BTL_USNIC_COMPAT_H
1417
#define BTL_USNIC_COMPAT_H
1518

19+
#include "opal/mca/rcache/rcache.h"
20+
1621
/************************************************************************/
1722

1823
/* v2.0 and beyond */
@@ -351,4 +356,18 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
351356

352357
#endif /* BTL_VERSION */
353358

359+
#if defined(RCACHE_MAJOR_VERSION) && RCACHE_MAJOR_VERSION > 3
360+
361+
#define RCACHE_VERSION 30
362+
363+
/* these structures got renamed with the mpool/rcache rewrite */
364+
#define mca_mpool_base_registration_t mca_rcache_base_registration_t
365+
#define mca_mpool_base_resources_t mca_rcache_base_resources_t
366+
367+
#else
368+
369+
#define RCACHE_VERSION 20
370+
371+
#endif
372+
354373
#endif /* BTL_USNIC_COMPAT_H */

‎opal/mca/btl/usnic/btl_usnic_mca.c‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,7 +13,7 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved.
15-
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
16+
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
1617
* reserved.
1718
* Copyright (c) 2015 Intel, Inc. All rights reserved.
1819
* $COPYRIGHT$
@@ -202,9 +203,19 @@ int opal_btl_usnic_component_register(void)
202203
0, &stats_relative, 0, OPAL_INFO_LVL_4));
203204
mca_btl_usnic_component.stats_relative = (bool) stats_relative;
204205

206+
#if RCACHE_VERSION == 30
207+
CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool",
208+
NULL, &mca_btl_usnic_component.usnic_mpool_hints, 0,
209+
OPAL_INFO_LVL_5));
210+
211+
CHECK(reg_string("rcache", "Name of the registration cache to be used",
212+
"grdma", &mca_btl_usnic_component.usnic_rcache_name, 0,
213+
OPAL_INFO_LVL_5));
214+
#else
205215
CHECK(reg_string("mpool", "Name of the memory pool to be used",
206216
"grdma", &mca_btl_usnic_component.usnic_mpool_name, 0,
207217
OPAL_INFO_LVL_5));
218+
#endif
208219

209220
want_numa_device_assignment = OPAL_HAVE_HWLOC ? 1 : -1;
210221
CHECK(reg_int("want_numa_device_assignment",

‎opal/mca/btl/usnic/btl_usnic_module.c‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* Copyright (c) 2006 Sandia National Laboratories. All rights
1414
* reserved.
1515
* Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved.
16-
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
16+
* Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
1717
* reserved.
1818
* Copyright (c) 2014 Intel, Inc. All rights reserved
1919
* $COPYRIGHT$
@@ -44,6 +44,8 @@
4444
#include "opal/mca/btl/base/btl_base_error.h"
4545
#include "opal/mca/mpool/base/base.h"
4646
#include "opal/mca/mpool/mpool.h"
47+
#include "opal/mca/rcache/base/base.h"
48+
#include "opal/mca/rcache/rcache.h"
4749
#else
4850
#include "ompi/mca/btl/btl.h"
4951
#include "ompi/mca/btl/base/btl_base_error.h"
@@ -805,7 +807,11 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
805807
OBJ_DESTRUCT(&module->chunk_segs);
806808
OBJ_DESTRUCT(&module->senders);
807809

810+
#if RCACHE_VERSION == 30
811+
mca_rcache_base_module_destroy(module->rcache);
812+
#else
808813
mca_mpool_base_module_destroy(module->super.btl_mpool);
814+
#endif
809815

810816
if (NULL != module->av) {
811817
fi_close(&module->av->fid);
@@ -1657,9 +1663,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
16571663
rd_num /* num erorments to alloc */,
16581664
rd_num /* max elements to alloc */,
16591665
rd_num /* num elements per alloc */,
1660-
module->super.btl_mpool /* mpool for reg */,
1666+
module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
16611667
0 /* mpool reg flags */,
1662-
NULL /* unused0 */,
1668+
module->rcache /* registration cache for 2.1+ */,
16631669
NULL /* item_init */,
16641670
NULL /* item_init_context */);
16651671
channel->recv_segs.ctx = module; /* must come after
@@ -1976,9 +1982,26 @@ static int init_mpool(opal_btl_usnic_module_t *module)
19761982
mpool_resources.deregister_mem = usnic_dereg_mr;
19771983
asprintf(&mpool_resources.pool_name, "%s",
19781984
module->fabric_info->fabric_attr->name);
1985+
#if RCACHE_VERSION == 30
1986+
module->rcache =
1987+
mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name,
1988+
&module->super, &mpool_resources);
1989+
if (NULL == module->rcache) {
1990+
opal_show_help("help-mpi-btl-usnic.txt",
1991+
"internal error during init",
1992+
true,
1993+
opal_process_info.nodename,
1994+
module->fabric_info->fabric_attr->name,
1995+
"create rcache", __FILE__, __LINE__);
1996+
return OPAL_ERROR;
1997+
}
1998+
module->super.btl_mpool =
1999+
mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints);
2000+
#else
19792001
module->super.btl_mpool =
19802002
mca_mpool_base_module_create(mca_btl_usnic_component.usnic_mpool_name,
19812003
&module->super, &mpool_resources);
2004+
#endif
19822005
if (NULL == module->super.btl_mpool) {
19832006
opal_show_help("help-mpi-btl-usnic.txt",
19842007
"internal error during init",
@@ -2149,7 +2172,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
21492172
module->sd_num / 2,
21502173
module->super.btl_mpool,
21512174
0 /* mpool reg flags */,
2152-
NULL /* unused0 */,
2175+
module->rcache,
21532176
NULL /* item_init */,
21542177
NULL /* item_init_context */);
21552178
assert(OPAL_SUCCESS == rc);
@@ -2204,7 +2227,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
22042227
module->sd_num / 2,
22052228
module->super.btl_mpool,
22062229
0 /* mpool reg flags */,
2207-
NULL /* unused0 */,
2230+
module->rcache,
22082231
NULL /* item_init */,
22092232
NULL /* item_init_context */);
22102233
assert(OPAL_SUCCESS == rc);
@@ -2226,7 +2249,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
22262249
module->sd_num / 2,
22272250
module->super.btl_mpool,
22282251
0 /* mpool reg flags */,
2229-
NULL /* unused0 */,
2252+
module->rcache,
22302253
NULL /* item_init */,
22312254
NULL /* item_init_context */);
22322255
assert(OPAL_SUCCESS == rc);
@@ -2283,7 +2306,11 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
22832306
int ret;
22842307
if (OPAL_SUCCESS != (ret = init_mpool(module)) ||
22852308
OPAL_SUCCESS != (ret = init_channels(module))) {
2309+
#if RCACHE_VERSION == 30
2310+
mca_rcache_base_module_destroy (module->rcache);
2311+
#else
22862312
mca_mpool_base_module_destroy(module->super.btl_mpool);
2313+
#endif
22872314
return ret;
22882315
}
22892316

‎opal/mca/btl/usnic/btl_usnic_module.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,6 +13,8 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2011-2015 Cisco Systems, Inc. All rights reserved.
16+
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
17+
* reserved.
1518
* $COPYRIGHT$
1619
*
1720
* Additional copyrights may follow
@@ -194,6 +197,9 @@ typedef struct opal_btl_usnic_module_t {
194197

195198
/* Performance / debugging statistics */
196199
opal_btl_usnic_module_stats_t stats;
200+
201+
/** registration cache module (v2.1+) */
202+
mca_rcache_base_module_t *rcache;
197203
} opal_btl_usnic_module_t;
198204

199205
struct opal_btl_usnic_frag_t;

0 commit comments

Comments
 (0)