From eac0b110b89db6e1c038e8392d025388b8b53662 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Fri, 18 Dec 2015 09:10:21 -0700 Subject: [PATCH] btl/usnic: update for mpool/rcache rewrite Signed-off-by: Nathan Hjelm --- opal/mca/btl/usnic/btl_usnic.h | 19 +++++++++++-- opal/mca/btl/usnic/btl_usnic_compat.h | 21 ++++++++++++++- opal/mca/btl/usnic/btl_usnic_mca.c | 13 ++++++++- opal/mca/btl/usnic/btl_usnic_module.c | 39 ++++++++++++++++++++++----- opal/mca/btl/usnic/btl_usnic_module.h | 6 +++++ 5 files changed, 88 insertions(+), 10 deletions(-) diff --git a/opal/mca/btl/usnic/btl_usnic.h b/opal/mca/btl/usnic/btl_usnic.h index cc094ce38f4..ddc264b3aaf 100644 --- a/opal/mca/btl/usnic/btl_usnic.h +++ b/opal/mca/btl/usnic/btl_usnic.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology @@ -12,6 +13,8 @@ * Copyright (c) 2006 Sandia National Laboratories. All rights * reserved. * Copyright (c) 2011-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -37,7 +40,13 @@ #include "opal/mca/btl/btl.h" #include "opal/mca/btl/base/btl_base_error.h" #include "opal/mca/btl/base/base.h" +#include "opal/mca/rcache/rcache.h" + +#include "btl_usnic_compat.h" + +#if RCACHE_VERSION < 30 #include "opal/mca/mpool/grdma/mpool_grdma.h" +#endif #else #include "ompi/mca/btl/btl.h" #include "ompi/mca/btl/base/btl_base_error.h" @@ -45,8 +54,6 @@ #include "ompi/mca/mpool/grdma/mpool_grdma.h" #endif -#include "btl_usnic_compat.h" - BEGIN_C_DECLS /* @@ -153,8 +160,16 @@ typedef struct opal_btl_usnic_component_t { /** list of usnic proc structures */ opal_list_t usnic_procs; +#if RCACHE_VERSION == 30 + /** memory pool hints */ + char* usnic_mpool_hints; + + /** registration cache name */ + char *usnic_rcache_name; +#else /** name of memory pool */ char* usnic_mpool_name; +#endif char *if_include; char *if_exclude; diff --git a/opal/mca/btl/usnic/btl_usnic_compat.h b/opal/mca/btl/usnic/btl_usnic_compat.h index 7a19a7e8515..da99d13be26 100644 --- a/opal/mca/btl/usnic/btl_usnic_compat.h +++ b/opal/mca/btl/usnic/btl_usnic_compat.h @@ -1,5 +1,8 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2013-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -8,11 +11,13 @@ */ /* This header contains macros to help minimize usnic BTL differences - * between v1.7/v1.8 and v1.9/v2.0. */ + * between v1.7/v1.8, v1.9/v2.0, and v2.0/v2.1. */ #ifndef BTL_USNIC_COMPAT_H #define BTL_USNIC_COMPAT_H +#include "opal/mca/rcache/rcache.h" + /************************************************************************/ /* v2.0 and beyond */ @@ -358,4 +363,18 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module, #endif /* BTL_VERSION */ +#if defined(RCACHE_MAJOR_VERSION) && RCACHE_MAJOR_VERSION >= 3 + +#define RCACHE_VERSION 30 + +/* these structures got renamed with the mpool/rcache rewrite */ +#define mca_mpool_base_registration_t mca_rcache_base_registration_t +#define mca_mpool_base_resources_t mca_rcache_base_resources_t + +#else + +#define RCACHE_VERSION 20 + +#endif + #endif /* BTL_USNIC_COMPAT_H */ diff --git a/opal/mca/btl/usnic/btl_usnic_mca.c b/opal/mca/btl/usnic/btl_usnic_mca.c index 910131d8f4f..006a32b6e69 100644 --- a/opal/mca/btl/usnic/btl_usnic_mca.c +++ b/opal/mca/btl/usnic/btl_usnic_mca.c @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology @@ -12,7 +13,7 @@ * Copyright (c) 2006 Sandia National Laboratories. All rights * reserved. * Copyright (c) 2008-2016 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved. * $COPYRIGHT$ @@ -203,9 +204,19 @@ int opal_btl_usnic_component_register(void) 0, &stats_relative, 0, OPAL_INFO_LVL_4)); mca_btl_usnic_component.stats_relative = (bool) stats_relative; +#if RCACHE_VERSION == 30 + CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool", + NULL, &mca_btl_usnic_component.usnic_mpool_hints, 0, + OPAL_INFO_LVL_5)); + + CHECK(reg_string("rcache", "Name of the registration cache to be used", + "grdma", &mca_btl_usnic_component.usnic_rcache_name, 0, + OPAL_INFO_LVL_5)); +#else CHECK(reg_string("mpool", "Name of the memory pool to be used", "grdma", &mca_btl_usnic_component.usnic_mpool_name, 0, OPAL_INFO_LVL_5)); +#endif want_numa_device_assignment = OPAL_HAVE_HWLOC ? 1 : -1; CHECK(reg_int("want_numa_device_assignment", diff --git a/opal/mca/btl/usnic/btl_usnic_module.c b/opal/mca/btl/usnic/btl_usnic_module.c index 713c3f44eb2..b8bf0ea114b 100644 --- a/opal/mca/btl/usnic/btl_usnic_module.c +++ b/opal/mca/btl/usnic/btl_usnic_module.c @@ -13,7 +13,7 @@ * Copyright (c) 2006 Sandia National Laboratories. All rights * reserved. * Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved * $COPYRIGHT$ @@ -44,6 +44,8 @@ #include "opal/mca/btl/base/btl_base_error.h" #include "opal/mca/mpool/base/base.h" #include "opal/mca/mpool/mpool.h" +#include "opal/mca/rcache/base/base.h" +#include "opal/mca/rcache/rcache.h" #else #include "ompi/mca/btl/btl.h" #include "ompi/mca/btl/base/btl_base_error.h" @@ -884,7 +886,11 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl) OBJ_DESTRUCT(&module->chunk_segs); OBJ_DESTRUCT(&module->senders); +#if RCACHE_VERSION == 30 + mca_rcache_base_module_destroy(module->rcache); +#else mca_mpool_base_module_destroy(module->super.btl_mpool); +#endif if (NULL != module->av) { fi_close(&module->av->fid); @@ -1736,9 +1742,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module, rd_num /* num erorments to alloc */, rd_num /* max elements to alloc */, rd_num /* num elements per alloc */, - module->super.btl_mpool /* mpool for reg */, + module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */, 0 /* mpool reg flags */, - NULL /* unused0 */, + module->rcache /* registration cache for 2.1+ */, NULL /* item_init */, NULL /* item_init_context */); channel->recv_segs.ctx = module; /* must come after @@ -2054,11 +2060,28 @@ static int init_mpool(opal_btl_usnic_module_t *module) mpool_resources.sizeof_reg = sizeof(opal_btl_usnic_reg_t); mpool_resources.register_mem = usnic_reg_mr; mpool_resources.deregister_mem = usnic_dereg_mr; +#if RCACHE_VERSION == 30 + module->rcache = + mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name, + &module->super, &mpool_resources); + if (NULL == module->rcache) { + opal_show_help("help-mpi-btl-usnic.txt", + "internal error during init", + true, + opal_process_info.nodename, + module->fabric_info->fabric_attr->name, + "create rcache", __FILE__, __LINE__); + return OPAL_ERROR; + } + module->super.btl_mpool = + mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints); +#else asprintf(&mpool_resources.pool_name, "%s", module->fabric_info->fabric_attr->name); module->super.btl_mpool = mca_mpool_base_module_create(mca_btl_usnic_component.usnic_mpool_name, &module->super, &mpool_resources); +#endif if (NULL == module->super.btl_mpool) { opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", @@ -2232,7 +2255,7 @@ static void init_freelists(opal_btl_usnic_module_t *module) module->sd_num / 2, module->super.btl_mpool, 0 /* mpool reg flags */, - NULL /* unused0 */, + module->rcache, NULL /* item_init */, NULL /* item_init_context */); assert(OPAL_SUCCESS == rc); @@ -2287,7 +2310,7 @@ static void init_freelists(opal_btl_usnic_module_t *module) module->sd_num / 2, module->super.btl_mpool, 0 /* mpool reg flags */, - NULL /* unused0 */, + module->rcache, NULL /* item_init */, NULL /* item_init_context */); assert(OPAL_SUCCESS == rc); @@ -2309,7 +2332,7 @@ static void init_freelists(opal_btl_usnic_module_t *module) module->sd_num / 2, module->super.btl_mpool, 0 /* mpool reg flags */, - NULL /* unused0 */, + module->rcache, NULL /* item_init */, NULL /* item_init_context */); assert(OPAL_SUCCESS == rc); @@ -2366,7 +2389,11 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module) int ret; if (OPAL_SUCCESS != (ret = init_mpool(module)) || OPAL_SUCCESS != (ret = init_channels(module))) { +#if RCACHE_VERSION == 30 + mca_rcache_base_module_destroy (module->rcache); +#else mca_mpool_base_module_destroy(module->super.btl_mpool); +#endif return ret; } diff --git a/opal/mca/btl/usnic/btl_usnic_module.h b/opal/mca/btl/usnic/btl_usnic_module.h index b4f5d0c7390..b7f49c596be 100644 --- a/opal/mca/btl/usnic/btl_usnic_module.h +++ b/opal/mca/btl/usnic/btl_usnic_module.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology @@ -12,6 +13,8 @@ * Copyright (c) 2006 Sandia National Laboratories. All rights * reserved. * Copyright (c) 2011-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -197,6 +200,9 @@ typedef struct opal_btl_usnic_module_t { /* Performance / debugging statistics */ opal_btl_usnic_module_stats_t stats; + + /** registration cache module (v2.1+) */ + mca_rcache_base_module_t *rcache; } opal_btl_usnic_module_t; struct opal_btl_usnic_frag_t;