fabric: Add mr_count field to domain attribute

Fixes ofiwg#3084

Allow the provider to indicate the number of memory registrations
that it can support on a domain.  This also allows the app a way
to indicate how many memory regions it may need.  Providers can
use this value to optimize caching of registrations.

Add missing output to fi_tostr as part of adding mr_cnt

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
Signed-off-by: Dmitry Gladkov <dmitry.gladkov@intel.com>

include/rdma/fabric.h

@@ -351,6 +351,7 @@ struct fi_domain_attr {
 	uint8_t			*auth_key;
 	size_t 			auth_key_size;
 	size_t			max_err_data;
+	size_t			mr_cnt;
 };
 
 struct fi_fabric_attr {

man/fi_domain.3.md

@@ -133,6 +133,7 @@ struct fi_domain_attr {
 	uint8_t               *auth_key;
 	size_t                auth_key_size;
 	size_t                max_err_data;
+	size_t                mr_cnt;
 };
 ```
 
@@ -638,6 +639,16 @@ opened with API version 1.5 or greater.
   a completion or event queue error.  This value corresponds to the
   err_data_size field in struct fi_cq_err_entry and struct fi_eq_err_entry.
 
+## Memory Regions Count (mr_cnt)
+
+The optimal number of memory regions supported by the domain.  The mr_cnt
+value may be a fixed value of the maximum number of MRs supported by the
+underlying hardware, or may be a dynamic value, based on the default
+attributes of the domain, such as the supported memory registration modes.
+Applications can set the mr_cnt on input to fi_getinfo, in order to
+indicate their memory registration requirements.  Doing so may allow the
+provider to optimize any memory registration cache or lookup tables.
+
 # RETURN VALUE
 
 Returns 0 on success. On error, a negative value corresponding to fabric

prov/util/src/util_attr.c

@@ -540,6 +540,18 @@ int ofi_check_domain_attr(const struct fi_provider *prov, uint32_t api_version,
 		return -FI_ENODATA;
 	}
 
+	if (user_attr->max_err_data > prov_attr->max_err_data) {
+		FI_INFO(prov, FI_LOG_CORE, "Max err data too large");
+		FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, max_err_data);
+		return -FI_ENODATA;
+	}
+
+	if (user_attr->mr_cnt > prov_attr->mr_cnt) {
+		FI_INFO(prov, FI_LOG_CORE, "MR count too large");
+		FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, mr_cnt);
+		return -FI_ENODATA;
+	}
+
 	return 0;
 }
 

src/fi_tostr.c

@@ -482,6 +482,19 @@ static void fi_tostr_domain_attr(char *buf, const struct fi_domain_attr *attr,
 	strcatf(buf, "%s%smax_ep_stx_ctx: %zd\n", prefix, TAB, attr->max_ep_stx_ctx);
 	strcatf(buf, "%s%smax_ep_srx_ctx: %zd\n", prefix, TAB, attr->max_ep_srx_ctx);
 	strcatf(buf, "%s%scntr_cnt: %zd\n", prefix, TAB, attr->cntr_cnt);
+	strcatf(buf, "%s%smr_iov_limit: %zd\n", prefix, TAB, attr->mr_iov_limit);
+
+	strcatf(buf, "%scaps: [ ", TAB);
+	fi_tostr_caps(buf, attr->caps);
+	strcatf(buf, " ]\n");
+
+	strcatf(buf, "%smode: [ ", TAB);
+	fi_tostr_mode(buf, attr->mode);
+	strcatf(buf, " ]\n");
+
+	strcatf(buf, "%s%sauth_key_size: %zd\n", prefix, TAB, attr->auth_key_size);
+	strcatf(buf, "%s%smax_err_data: %zd\n", prefix, TAB, attr->max_err_data);
+	strcatf(buf, "%s%smr_cnt: %zd\n", prefix, TAB, attr->mr_cnt);
 }
 
 static void fi_tostr_fabric_attr(char *buf, const struct fi_fabric_attr *attr,