Skip to content

Commit

Permalink
Merge pull request ofiwg#13 from bturrubiates/topic/fi-source
Browse files Browse the repository at this point in the history 
Topic/fi source
  • Loading branch information
@hppritcha
hppritcha committed Feb 17, 2015
2 parents 7c1f3a3 + 1ba07fc commit 51899ce
Showing 1 changed file with 106 additions and 87 deletions.
193 changes: 106 additions & 87 deletions prov/gni/src/gnix_fabric.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
*/

#if HAVE_CONFIG_H
# include <config.h>
#include <config.h>
#endif /* HAVE_CONFIG_H */

#include <errno.h>
Expand Down Expand Up @@ -67,9 +67,9 @@ const struct fi_fabric_attr gnix_fabric_attr = {
static struct fi_ops_fabric gnix_fab_ops = {
.size = sizeof(struct fi_ops_fabric),
.domain = gnix_domain_open,
.passive_ep = NULL, /* TODO: need to define for FI_EP_MSG */
.eq_open = NULL, /* TODO: need to define for FI_EP_MSG */
.wait_open = NULL, /* TODO: what's this about */
.passive_ep = NULL, /* TODO: need to define for FI_EP_MSG */
.eq_open = NULL, /* TODO: need to define for FI_EP_MSG */
.wait_open = NULL, /* TODO: what's this about */
};

static int gnix_fabric_close(fid_t fid)
Expand Down Expand Up @@ -97,8 +97,8 @@ static struct fi_ops gnix_fab_fi_ops = {
* define methods needed for the GNI fabric provider
*/

static int gnix_fabric(struct fi_fabric_attr *attr,
struct fid_fabric **fabric, void *context)
static int gnix_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
void *context)
{
struct gnix_fabric *fab;

Expand All @@ -121,55 +121,64 @@ static int gnix_fabric(struct fi_fabric_attr *attr,
}

static int gnix_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
uint64_t flags, struct fi_info *hints,
struct fi_info **info)
{
int ret = 0;
int mode = GNIX_FAB_MODES;
struct fi_info *gnix_info;
struct gnix_ep_name *dest_addr = NULL;

/*
* the code below for resolving a node/service to what
* will be a gnix_ep_name address is not fully implemented,
* but put a place holder in place
*/

if (node && !(flags & FI_SOURCE)) {
dest_addr = (struct gnix_ep_name *)malloc(sizeof(*dest_addr));
if (dest_addr == NULL) {
ret = -FI_ENOMEM;
struct gnix_ep_name *dest_addr = NULL, *src_addr = NULL, *addr = NULL;

/*
* the code below for resolving a node/service to what
* will be a gnix_ep_name address is not fully implemented,
* but put a place holder in place
*/
if (node) {
addr = malloc(sizeof(*addr));
if (!addr) {
ret = -FI_ENOMEM;
goto err;
}
ret = gnix_resolve_name(node,service,dest_addr);
if (ret) goto err;

ret = gnix_resolve_name(node, service, addr);
if (ret) {
goto err;
}

if (flags & FI_SOURCE) {
src_addr = addr;
} else {
dest_addr = addr;
}
}

if (hints) {
if (hints) {

/*
* check for endpoint type, only support FI_EP_RDM for now
*/
/*
* check for endpoint type, only support FI_EP_RDM for now
*/

switch (hints->ep_type) {
case FI_EP_UNSPEC:
case FI_EP_RDM:
break;
default:
switch (hints->ep_type) {
case FI_EP_UNSPEC:
case FI_EP_RDM:
break;
default:
ret = -FI_ENODATA;
goto err;
}
goto err;
}

/*
* check the mode field
*/

if (hints->mode) {
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
ret = -FI_ENODATA;
goto err;
}
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
ret = -FI_ENODATA;
goto err;
}
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
}

if ((hints->caps & GNIX_EP_RDM_CAPS) != hints->caps) {
goto err;
Expand All @@ -193,51 +202,53 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
if (hints->ep_attr->rx_ctx_cnt > 1) {
ret = -FI_ENODATA;
goto err;
}
}
}
}

if (hints->tx_attr &&
(hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
(hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
ret = -FI_ENODATA;
goto err;
}
}

if (hints->rx_attr &&
(hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
(hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->rx_attr->op_flags) {
ret = -FI_ENODATA;
goto err;
}

if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, gnix_fab_name, strlen(gnix_fab_name))) {
strncmp(hints->fabric_attr->name, gnix_fab_name,
strlen(gnix_fab_name))) {
ret = -FI_ENODATA;
goto err;
}

/* TODO: use hardwared kgni const string */
if (hints->domain_attr && hints->domain_attr->name &&
strncmp(hints->domain_attr->name, gnix_dom_name, strlen(gnix_dom_name))) {
ret = -FI_ENODATA;
goto err;
strncmp(hints->domain_attr->name, gnix_dom_name,
strlen(gnix_dom_name))) {
ret = -FI_ENODATA;
goto err;
}

if (hints->ep_attr) {
if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE) {
ret = -FI_ENODATA;
goto err;
}
if (hints->ep_attr->inject_size > GNIX_INJECT_SIZE) {
ret = -FI_ENODATA;
goto err;
}
/*
* TODO: tag matching
max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format);
*/
}

if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE) {
ret = -FI_ENODATA;
goto err;
}
if (hints->ep_attr->inject_size > GNIX_INJECT_SIZE) {
ret = -FI_ENODATA;
goto err;
}
/*
* TODO: tag matching
max_tag_value =
fi_tag_bits(hints->ep_attr->mem_tag_format);
*/
}
}

/*
Expand All @@ -247,54 +258,66 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
gnix_info = fi_allocinfo_internal();
if (gnix_info == NULL) {
ret = -FI_ENOMEM;
goto err;
goto err;
}

gnix_info->ep_attr->protocol = FI_PROTO_GNI;
gnix_info->ep_attr->max_msg_size = GNIX_MAX_MSG_SIZE;
gnix_info->ep_attr->inject_size = GNIX_INJECT_SIZE;
gnix_info->ep_attr->total_buffered_recv = ~(0ULL); /* TODO: need to work on this */
gnix_info->ep_attr->mem_tag_format = 0x0; /* TODO: need to work on this */
gnix_info->ep_attr->msg_order = FI_ORDER_SAS; /* TODO: remember this when implementing sends */
gnix_info->ep_attr->total_buffered_recv =
~(0ULL); /* TODO: need to work on this */
gnix_info->ep_attr->mem_tag_format =
0x0; /* TODO: need to work on this */
gnix_info->ep_attr->msg_order =
FI_ORDER_SAS; /* TODO: remember this when implementing sends */
gnix_info->ep_attr->comp_order = FI_ORDER_NONE;
gnix_info->ep_attr->tx_ctx_cnt = 1;
gnix_info->ep_attr->rx_ctx_cnt = 1;

gnix_info->domain_attr->threading = FI_THREAD_COMPLETION;
gnix_info->domain_attr->control_progress = FI_PROGRESS_AUTO;
gnix_info->domain_attr->data_progress = FI_PROGRESS_AUTO;
gnix_info->domain_attr->name = strdup(gnix_dom_name); /* only one aries per node */
gnix_info->domain_attr->name =
strdup(gnix_dom_name); /* only one aries per node */

gnix_info->next = NULL;
gnix_info->ep_type = FI_EP_RDM;
gnix_info->caps = GNIX_EP_RDM_CAPS;
gnix_info->mode = mode;
gnix_info->addr_format = FI_ADDR_GNI;
gnix_info->src_addrlen = 0;
gnix_info->src_addrlen = sizeof(struct gnix_ep_name);
gnix_info->dest_addrlen = sizeof(struct gnix_ep_name);
gnix_info->src_addr = NULL;
gnix_info->src_addr = src_addr;
gnix_info->dest_addr = dest_addr;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
gnix_info->fabric_attr->prov_name = strdup(gnix_fab_name); /* let's consider gni copyrighted :) */
gnix_info->fabric_attr->prov_name =
strdup(gnix_fab_name); /* let's consider gni copyrighted :) */

gnix_info->tx_attr->caps = gnix_info->caps;
gnix_info->tx_attr->mode = gnix_info->mode;
gnix_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : GNIX_EP_OP_FLAGS;
gnix_info->tx_attr->op_flags =
(hints && hints->tx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags
: GNIX_EP_OP_FLAGS;
gnix_info->tx_attr->msg_order = gnix_info->ep_attr->msg_order;
gnix_info->tx_attr->comp_order = gnix_info->ep_attr->comp_order;
gnix_info->tx_attr->inject_size = gnix_info->ep_attr->inject_size;
gnix_info->tx_attr->size = UINT64_MAX; /* TODO: probably something else here */
gnix_info->tx_attr->size =
UINT64_MAX; /* TODO: probably something else here */
gnix_info->tx_attr->iov_limit = 1;

gnix_info->rx_attr->caps = gnix_info->caps;
gnix_info->rx_attr->mode = gnix_info->mode;
gnix_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : GNIX_EP_OP_FLAGS;
gnix_info->rx_attr->op_flags =
(hints && hints->rx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags
: GNIX_EP_OP_FLAGS;
gnix_info->rx_attr->msg_order = gnix_info->ep_attr->msg_order;
gnix_info->rx_attr->comp_order = gnix_info->ep_attr->comp_order;
gnix_info->rx_attr->total_buffered_recv = gnix_info->ep_attr->total_buffered_recv;
gnix_info->rx_attr->size = UINT64_MAX; /* TODO: probably something else here */
gnix_info->rx_attr->total_buffered_recv =
gnix_info->ep_attr->total_buffered_recv;
gnix_info->rx_attr->size =
UINT64_MAX; /* TODO: probably something else here */
gnix_info->rx_attr->iov_limit = 1;

*info = gnix_info;
Expand All @@ -303,21 +326,17 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
return ret;
}



static void gnix_fini(void)
{
}

struct fi_provider gnix_prov = {
.name = "gni",
.version = FI_VERSION(GNI_MAJOR_VERSION, GNI_MINOR_VERSION),
.version = FI_VERSION(GNI_MAJOR_VERSION, GNI_MINOR_VERSION),
.fi_version = FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION),
.getinfo = gnix_getinfo,
.fabric = gnix_fabric,
.cleanup = gnix_fini
};

.cleanup = gnix_fini};

GNI_INI
{
Expand Down Expand Up @@ -347,19 +366,19 @@ GNI_INI
return NULL;
}

assert(num_devices == 1); /* sanity check that the 1 aries/node holds */
assert(num_devices == 1); /* sanity check that the 1 aries/node holds */

/*
* don't register if available ugni is older than one libfabric was built against
* don't register if available ugni is older than one libfabric was
* built against
*/

status = GNI_GetVersionInformation(&lib_version);
if ((GNI_GET_MAJOR(lib_version.ugni_version) > GNI_MAJOR_REV) ||
((GNI_GET_MAJOR(lib_version.ugni_version) == GNI_MAJOR_REV) &&
GNI_GET_MINOR(lib_version.ugni_version) >= GNI_MINOR_REV)) {
((GNI_GET_MAJOR(lib_version.ugni_version) == GNI_MAJOR_REV) &&
GNI_GET_MINOR(lib_version.ugni_version) >= GNI_MINOR_REV)) {
provider = &gnix_prov;
}

fprintf(stderr,"returning provider %p\n",provider);
return (provider);
}

0 comments on commit 51899ce

Please sign in to comment.