Merge pull request #7788 from wzamazon/efa_info_squash

prov/efa: info squash

libfabric.vcxproj

@@ -857,6 +857,8 @@
     <ClCompile Include="prov\efa\src\efa_mr.c" />
     <ClCompile Include="prov\efa\src\efa_msg.c" />
     <ClCompile Include="prov\efa\src\efa_rma.c" />
+    <ClCompile Include="prov\efa\src\efa_shm.c" />
+    <ClCompile Include="prov\efa\src\efa_user_info.c" />
     <ClCompile Include="prov\efa\src\efa_prov_info.c" />
     <ClCompile Include="prov\efa\src\efa_fork_support.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_pkt_type_base.c" />
@@ -866,7 +868,7 @@
     <ClCompile Include="prov\efa\src\rxr\rxr_cntr.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_cq.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_ep.c" />
-    <ClCompile Include="prov\efa\src\rxr\rxr_init.c" />
+    <ClCompile Include="prov\efa\src\rxr\rxr_env.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_msg.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_pkt_cmd.c" />
     <ClCompile Include="prov\efa\src\rxr\rxr_pkt_entry.c" />

prov/efa/Makefile.include

@@ -33,6 +33,7 @@ if HAVE_EFA
 _efa_files = \
 	prov/efa/src/efa_device.c \
 	prov/efa/src/efa_hmem.c \
+	prov/efa/src/efa_shm.c \
 	prov/efa/src/efa_av.c \
 	prov/efa/src/efa_domain.c \
 	prov/efa/src/efa_cm.c \
@@ -42,10 +43,11 @@ _efa_files = \
 	prov/efa/src/efa_msg.c \
 	prov/efa/src/efa_mr.c \
 	prov/efa/src/efa_rma.c \
+	prov/efa/src/efa_user_info.c \
 	prov/efa/src/efa_prov_info.c \
 	prov/efa/src/efa_fork_support.c \
 	prov/efa/src/rxr/rxr_prov.c	\
-	prov/efa/src/rxr/rxr_init.c	\
+	prov/efa/src/rxr/rxr_env.c	\
 	prov/efa/src/rxr/rxr_cq.c	\
 	prov/efa/src/rxr/rxr_ep.c	\
 	prov/efa/src/rxr/rxr_cntr.c	\
@@ -64,13 +66,15 @@ _efa_files = \
 _efa_headers = \
 	prov/efa/src/efa.h \
 	prov/efa/src/efa_mr.h \
+	prov/efa/src/efa_shm.h \
 	prov/efa/src/efa_hmem.h \
 	prov/efa/src/efa_device.h \
 	prov/efa/src/efa_domain.h \
+	prov/efa/src/efa_user_info.h \
 	prov/efa/src/efa_prov_info.h \
 	prov/efa/src/efa_fork_support.h \
 	prov/efa/src/rxr/rxr.h \
-	prov/efa/src/rxr/rxr_init.h \
+	prov/efa/src/rxr/rxr_env.h \
 	prov/efa/src/rxr/rxr_cntr.h \
 	prov/efa/src/rxr/rxr_rma.h \
 	prov/efa/src/rxr/rxr_msg.h \

prov/efa/src/efa.h

@@ -65,9 +65,11 @@
 #include "ofi_file.h"
 
 #include "efa_mr.h"
+#include "efa_shm.h"
 #include "efa_hmem.h"
 #include "efa_device.h"
 #include "efa_domain.h"
+#include "efa_user_info.h"
 #include "efa_fork_support.h"
 #include "rxr.h"
 #define EFA_PROV_NAME "efa"
@@ -139,6 +141,26 @@ struct efa_ah {
 	UT_hash_handle	hh; /* hash map handle, link all efa_ah with efa_ep->ah_map */
 };
 
+static inline
+int efa_str_to_ep_addr(const char *node, const char *service, struct efa_ep_addr *addr)
+{
+	int ret;
+
+	if (!node)
+		return -FI_EINVAL;
+
+	memset(addr, 0, sizeof(*addr));
+
+	ret = inet_pton(AF_INET6, node, addr->raw);
+	if (ret != 1)
+		return -FI_EINVAL;
+	if (service)
+		addr->qpn = atoi(service);
+
+	return 0;
+}
+
+
 struct efa_conn {
 	struct efa_ah		*ah;
 	struct efa_ep_addr	*ep_addr;
@@ -284,8 +306,6 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
 		struct fid_cq **cq_fid, void *context);
 int efa_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric_fid,
 	       void *context);
-int efa_getinfo(uint32_t version, const char *node, const char *service,
-		uint64_t flags, const struct fi_info *hints, struct fi_info **info);
 
 /* AV sub-functions */
 int efa_av_insert_one(struct efa_av *av, struct efa_ep_addr *addr,

prov/efa/src/efa_av.c

@@ -407,7 +407,7 @@ int efa_conn_rdm_init(struct efa_av *av, struct efa_conn *conn)
 		}
 
 		smr_name_len = EFA_SHM_NAME_MAX;
-		err = rxr_raw_addr_to_smr_name(conn->ep_addr, smr_name, &smr_name_len);
+		err = efa_shm_ep_name_construct(smr_name, &smr_name_len, conn->ep_addr);
 		if (err != FI_SUCCESS) {
 			EFA_WARN(FI_LOG_AV,
 				 "rxr_ep_efa_addr_to_str() failed! err=%d\n", err);

prov/efa/src/efa_domain.c

@@ -124,8 +124,8 @@ static int efa_domain_init_rdm(struct efa_domain *efa_domain, struct fi_info *in
 	int err;
 
 	if (efa_domain->fabric->shm_fabric) {
-		assert(!strcmp(shm_info->fabric_attr->name, "shm"));
-		err = fi_domain(efa_domain->fabric->shm_fabric, shm_info,
+		assert(!strcmp(g_shm_info->fabric_attr->name, "shm"));
+		err = fi_domain(efa_domain->fabric->shm_fabric, g_shm_info,
 				&efa_domain->shm_domain, NULL);
 		if (err)
 			return err;

prov/efa/src/efa_fabric.c

@@ -53,6 +53,7 @@
 
 #include "efa.h"
 #include "efa_prov_info.h"
+
 #if HAVE_EFA_DL
 #include <ofi_shm.h>
 #endif
@@ -63,229 +64,6 @@ const char *efa_perf_counters_str[] = {
 };
 #endif
 
-static int efa_check_hints(uint32_t version, const struct fi_info *hints,
-			   const struct fi_info *info)
-{
-	uint64_t prov_mode;
-	size_t size;
-	int ret;
-
-	if (hints->caps & ~(info->caps)) {
-		EFA_INFO(FI_LOG_CORE, "Unsupported capabilities\n");
-		OFI_INFO_CHECK(&efa_prov, info, hints, caps, FI_TYPE_CAPS);
-		return -FI_ENODATA;
-	}
-
-	prov_mode = ofi_mr_get_prov_mode(version, hints, info);
-
-	if ((hints->mode & prov_mode) != prov_mode) {
-		EFA_INFO(FI_LOG_CORE, "Required hints mode bits not set\n");
-		OFI_INFO_MODE(&efa_prov, prov_mode, hints->mode);
-		return -FI_ENODATA;
-	}
-
-	if (hints->fabric_attr) {
-		ret = ofi_check_fabric_attr(&efa_prov, info->fabric_attr,
-					    hints->fabric_attr);
-
-		if (ret)
-			return ret;
-	}
-
-	switch (hints->addr_format) {
-	case FI_FORMAT_UNSPEC:
-	case FI_ADDR_EFA:
-		size = EFA_EP_ADDR_LEN;
-		break;
-	default:
-		EFA_INFO(FI_LOG_CORE,
-			 "Address format not supported: hints[%u], supported[%u,%u]\n",
-			 hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_EFA);
-		return -FI_ENODATA;
-	}
-
-	if (hints->src_addr && hints->src_addrlen < size)
-		return -FI_ENODATA;
-
-	if (hints->dest_addr && hints->dest_addrlen < size)
-		return -FI_ENODATA;
-
-	if (hints->domain_attr) {
-		ret = ofi_check_domain_attr(&efa_prov, version, info->domain_attr, hints);
-		if (ret)
-			return ret;
-	}
-
-	if (hints->ep_attr) {
-		ret = ofi_check_ep_attr(&efa_util_prov, info->fabric_attr->api_version, info, hints);
-		if (ret)
-			return ret;
-	}
-
-	if (hints->rx_attr) {
-		ret = ofi_check_rx_attr(&efa_prov, info, hints->rx_attr, hints->mode);
-		if (ret)
-			return ret;
-	}
-
-	if (hints->tx_attr) {
-		ret = ofi_check_tx_attr(&efa_prov, info->tx_attr, hints->tx_attr, hints->mode);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int efa_str_to_ep_addr(const char *node, const char *service, struct efa_ep_addr *addr)
-{
-	int ret;
-
-	if (!node)
-		return -FI_EINVAL;
-
-	memset(addr, 0, sizeof(*addr));
-
-	ret = inet_pton(AF_INET6, node, addr->raw);
-	if (ret != 1)
-		return -FI_EINVAL;
-	if (service)
-		addr->qpn = atoi(service);
-
-	return 0;
-}
-
-static int efa_node_matches_addr(struct efa_ep_addr *addr, const char *node)
-{
-	struct efa_ep_addr eaddr;
-
-	efa_str_to_ep_addr(node, NULL, &eaddr);
-	return memcmp(&eaddr.raw, &addr->raw, sizeof(addr->raw));
-}
-
-static int efa_get_matching_info(uint32_t version, const char *node, uint64_t flags,
-				 const struct fi_info *hints, struct fi_info **info)
-{
-	const struct fi_info *check_info;
-	struct fi_info *fi, *tail;
-	int ret;
-
-	*info = tail = NULL;
-
-	for (check_info = efa_util_prov.info; check_info; check_info = check_info->next) {
-		ret = 0;
-		if (flags & FI_SOURCE) {
-			if (node)
-				ret = efa_node_matches_addr(check_info->src_addr, node);
-		} else if (hints && hints->src_addr) {
-			ret = memcmp(check_info->src_addr, hints->src_addr, EFA_EP_ADDR_LEN);
-		}
-
-		if (ret)
-			continue;
-		EFA_INFO(FI_LOG_FABRIC, "found match for interface %s %s\n", node, check_info->fabric_attr->name);
-		if (hints) {
-			ret = efa_check_hints(version, hints, check_info);
-			if (ret)
-				continue;
-		}
-
-		fi = fi_dupinfo(check_info);
-		if (!fi) {
-			ret = -FI_ENOMEM;
-			goto err_free_info;
-		}
-
-		fi->fabric_attr->api_version = version;
-
-		if (!*info)
-			*info = fi;
-		else
-			tail->next = fi;
-		tail = fi;
-	}
-
-	if (!*info)
-		return -FI_ENODATA;
-
-	return 0;
-
-err_free_info:
-	fi_freeinfo(*info);
-	*info = NULL;
-	return ret;
-}
-
-static int efa_set_fi_address(const char *node, const char *service, uint64_t flags,
-			      const struct fi_info *hints, struct fi_info *fi)
-{
-	struct efa_ep_addr tmp_addr;
-	void *dest_addr = NULL;
-	int ret = FI_SUCCESS;
-	struct fi_info *cur;
-
-	if (flags & FI_SOURCE) {
-		if (hints && hints->dest_addr)
-			dest_addr = hints->dest_addr;
-	} else {
-		if (node || service) {
-			ret = efa_str_to_ep_addr(node, service, &tmp_addr);
-			if (ret)
-				return ret;
-			dest_addr = &tmp_addr;
-		} else if (hints && hints->dest_addr) {
-			dest_addr = hints->dest_addr;
-		}
-	}
-
-	if (dest_addr) {
-		for (cur = fi; cur; cur = cur->next) {
-			cur->dest_addr = malloc(EFA_EP_ADDR_LEN);
-			if (!cur->dest_addr) {
-				for (; fi->dest_addr; fi = fi->next)
-					free(fi->dest_addr);
-				return -FI_ENOMEM;
-			}
-			memcpy(cur->dest_addr, dest_addr, EFA_EP_ADDR_LEN);
-			cur->dest_addrlen = EFA_EP_ADDR_LEN;
-		}
-	}
-	return ret;
-}
-
-int efa_getinfo(uint32_t version, const char *node, const char *service,
-		uint64_t flags, const struct fi_info *hints, struct fi_info **info)
-{
-	int ret;
-
-	if (!(flags & FI_SOURCE) && hints && hints->src_addr &&
-	    hints->src_addrlen != EFA_EP_ADDR_LEN)
-		return -FI_ENODATA;
-
-	if (((!node && !service) || (flags & FI_SOURCE)) &&
-	    hints && hints->dest_addr &&
-	    hints->dest_addrlen != EFA_EP_ADDR_LEN)
-		return -FI_ENODATA;
-
-	ret = efa_get_matching_info(version, node, flags, hints, info);
-	if (ret)
-		goto out;
-
-	ret = efa_set_fi_address(node, service, flags, hints, *info);
-	if (ret)
-		goto out;
-
-	ofi_alter_info(*info, hints, version);
-
-out:
-	if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV) {
-		return ret;
-	} else {
-		fi_freeinfo(*info);
-		*info = NULL;
-		return -FI_ENODATA;
-	}
-}
 
 static int efa_fabric_close(fid_t fid)
 {
@@ -359,9 +137,9 @@ int efa_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric_fid,
 		goto err_free_fabric;
 
 	/* Open shm provider's fabric domain */
-	if (shm_info) {
-		assert(!strcmp(shm_info->fabric_attr->name, "shm"));
-		ret = fi_fabric(shm_info->fabric_attr,
+	if (g_shm_info) {
+		assert(!strcmp(g_shm_info->fabric_attr->name, "shm"));
+		ret = fi_fabric(g_shm_info->fabric_attr,
 				&efa_fabric->shm_fabric, context);
 		if (ret)
 			goto err_close_util_fabric;