btl/scif: update for new rcache and mpool interfaces

Signed-off-by: Nathan Hjelm <hjelmn@me.com>

Author: hjelmn

opal/mca/btl/scif/btl_scif.h

@@ -18,7 +18,7 @@
 
 #include "opal/mca/mpool/mpool.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/mca/mpool/grdma/mpool_grdma.h"
+#include "opal/mca/rcache/rcache.h"
 #include "opal/util/output.h"
 #include "opal_stdint.h"
 #include "opal/util/proc.h"
@@ -93,6 +93,9 @@ typedef struct mca_btl_scif_module_t {
 
     volatile bool exiting;
     bool listening;
+
+    /** registration cache */
+    mca_rcache_base_module_t *rcache;
 } mca_btl_scif_module_t;
 
 typedef struct mca_btl_scif_component_t {
@@ -235,7 +238,7 @@ struct mca_btl_scif_registration_handle_t {
 typedef struct mca_btl_scif_registration_handle_t mca_btl_scif_registration_handle_t;
 
 typedef struct mca_btl_scif_reg_t {
-    mca_mpool_base_registration_t base;
+    mca_rcache_base_registration_t base;
     /** per-endpoint btl handles for this registration */
     mca_btl_scif_registration_handle_t *handles;
 } mca_btl_scif_reg_t;

opal/mca/btl/scif/btl_scif_add_procs.c

@@ -157,7 +157,7 @@ int mca_btl_scif_del_procs (struct mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
+static int scif_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
 {
     mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
     size_t size = (size_t)((uintptr_t) reg->bound - (uintptr_t) reg->base);
@@ -178,15 +178,15 @@ static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
 }
 
 static int scif_reg_mem (void *reg_data, void *base, size_t size,
-                         mca_mpool_base_registration_t *reg)
+                         mca_rcache_base_registration_t *reg)
 {
     mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
     int rc = OPAL_SUCCESS;
     unsigned int i;
 
     scif_reg->handles = calloc (mca_btl_scif_module.endpoint_count, sizeof (scif_reg->handles[0]));
 
-    /* intialize all scif offsets to -1 and initialize the pointer back to the mpool registration */
+    /* intialize all scif offsets to -1 and initialize the pointer back to the rcache registration */
     for (i = 0 ; i < mca_btl_scif_module.endpoint_count ; ++i) {
         scif_reg->handles[i].btl_handle.scif_offset = -1;
         scif_reg->handles[i].btl_handle.scif_base = (intptr_t) base;
@@ -214,19 +214,19 @@ static int scif_reg_mem (void *reg_data, void *base, size_t size,
 static int
 mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
 {
-    struct mca_mpool_base_resources_t mpool_resources;
+    mca_rcache_base_resources_t rcache_resources;
     int rc;
 
-    /* initialize the grdma mpool */
-    mpool_resources.pool_name      = "scif";
-    mpool_resources.reg_data       = (void *) scif_module;
-    mpool_resources.sizeof_reg     = sizeof (mca_btl_scif_reg_t);
-    mpool_resources.register_mem   = scif_reg_mem;
-    mpool_resources.deregister_mem = scif_dereg_mem;
-    scif_module->super.btl_mpool =
-        mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
-    if (NULL == scif_module->super.btl_mpool) {
-        BTL_ERROR(("error creating grdma mpool"));
+    /* initialize the grdma rcache */
+    rcache_resources.pool_name      = "scif";
+    rcache_resources.reg_data       = (void *) scif_module;
+    rcache_resources.sizeof_reg     = sizeof (mca_btl_scif_reg_t);
+    rcache_resources.register_mem   = scif_reg_mem;
+    rcache_resources.deregister_mem = scif_dereg_mem;
+    scif_module->rcache =
+        mca_rcache_base_module_create("grdma", scif_module, &rcache_resources);
+    if (NULL == scif_module->rcache) {
+        BTL_ERROR(("error creating grdma rcache"));
         return OPAL_ERROR;
     }
 

opal/mca/btl/scif/btl_scif_frag.h

@@ -60,8 +60,8 @@ static inline int mca_btl_scif_frag_alloc (mca_btl_base_endpoint_t *ep,
 static inline int mca_btl_scif_frag_return (mca_btl_scif_base_frag_t *frag)
 {
     if (frag->registration) {
-        frag->endpoint->btl->super.btl_mpool->mpool_deregister(frag->endpoint->btl->super.btl_mpool,
-                                                               &frag->registration->base);
+        frag->endpoint->btl->rcache->rcache_deregister(frag->endpoint->btl->rcache,
+                                                       &frag->registration->base);
         frag->registration = NULL;
     }
 

opal/mca/btl/scif/btl_scif_module.c

@@ -106,6 +106,11 @@ mca_btl_scif_module_finalize (struct mca_btl_base_module_t *btl)
     mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
     unsigned int i;
 
+    if (scif_module->rcache) {
+        mca_rcache_base_module_destroy (scif_module->rcache);
+        scif_module->rcache = NULL;
+    }
+
     OBJ_DESTRUCT(&mca_btl_scif_module.dma_frags);
     OBJ_DESTRUCT(&mca_btl_scif_module.eager_frags);
 
@@ -180,6 +185,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
                                                                       mca_btl_base_endpoint_t *endpoint,
                                                                       void *base, size_t size, uint32_t flags)
 {
+    mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
     mca_btl_scif_reg_t *scif_reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rc;
@@ -200,8 +206,8 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
         }
     }
 
-    rc = btl->btl_mpool->mpool_register(btl->btl_mpool, base, size, 0, access_flags,
-                                        (mca_mpool_base_registration_t **) &scif_reg);
+    rc = scif_module->rcache->rcache_register (scif_module->rcache, base, size, 0, access_flags,
+                                               (mca_rcache_base_registration_t **) &scif_reg);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return NULL;
     }
@@ -210,7 +216,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
     if ((off_t) -1 == scif_reg->handles[endpoint->id].btl_handle.scif_offset) {
         size_t seg_size = (size_t)((uintptr_t) scif_reg->base.bound - (uintptr_t) scif_reg->base.base) + 1;
 
-        /* NTH: until we determine a way to pass permissions to the mpool just make all segments
+        /* NTH: until we determine a way to pass permissions to the rcache just make all segments
          * read/write */
         scif_reg->handles[endpoint->id].btl_handle.scif_offset =
             scif_register (endpoint->scif_epd, scif_reg->base.base, seg_size, 0, SCIF_PROT_READ |
@@ -224,10 +230,11 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
 
 static int mca_btl_scif_deregister_mem (struct mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
 {
+    mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
     mca_btl_scif_registration_handle_t *scif_handle = (mca_btl_scif_registration_handle_t *) handle;
     mca_btl_scif_reg_t *scif_reg = scif_handle->reg;
 
-    btl->btl_mpool->mpool_deregister (btl->btl_mpool, &scif_reg->base);
+    scif_module->rcache->rcache_deregister (scif_module->rcache, &scif_reg->base);
 
     return OPAL_SUCCESS;
 }