fabric: Allow reuse of fi_info structure

The API is currently defined such that an fi_info structure
can only be used to open a single endpoint.  Update the
API to support re-using the fi_info structure.

Update the verbs provider to support this as well.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

include/rdma/fabric.h

@@ -125,6 +125,7 @@ enum fi_addr_format {
 
 #define FI_ADDR_UNSPEC		UINT64_MAX
 typedef uint64_t		fi_addr_t;
+typedef void *			fi_connreq_t;
 
 enum fi_progress {
 	FI_PROGRESS_UNSPEC,
@@ -193,11 +194,10 @@ struct fi_info {
 	size_t			dest_addrlen;
 	void			*src_addr;
 	void			*dest_addr;
+	fi_connreq_t		connreq;
 	struct fi_ep_attr	*ep_attr;
 	struct fi_domain_attr	*domain_attr;
 	struct fi_fabric_attr	*fabric_attr;
-	size_t			datalen;
-	void			*data;
 };
 
 enum {

include/rdma/fi_cm.h

@@ -48,8 +48,7 @@ struct fi_ops_cm {
 	int	(*connect)(struct fid_ep *ep, const void *addr,
 			const void *param, size_t paramlen);
 	int	(*listen)(struct fid_pep *pep);
-	int	(*accept)(struct fid_ep *ep, fi_connreq_t connreq,
-			const void *param, size_t paramlen);
+	int	(*accept)(struct fid_ep *ep, const void *param, size_t paramlen);
 	int	(*reject)(struct fid_pep *pep, fi_connreq_t connreq,
 			const void *param, size_t paramlen);
 	int	(*shutdown)(struct fid_ep *ep, uint64_t flags);
@@ -86,10 +85,9 @@ fi_connect(struct fid_ep *ep, const void *addr,
 }
 
 static inline int
-fi_accept(struct fid_ep *ep, fi_connreq_t connreq,
-	  const void *param, size_t paramlen)
+fi_accept(struct fid_ep *ep, const void *param, size_t paramlen)
 {
-	return ep->cm->accept(ep, connreq, param, paramlen);
+	return ep->cm->accept(ep, param, paramlen);
 }
 
 static inline int

include/rdma/fi_eq.h

@@ -131,11 +131,8 @@ struct fi_eq_err_entry {
 	void			*err_data;
 };
 
-typedef void *			fi_connreq_t;
-
 struct fi_eq_cm_entry {
 	fid_t			fid;
-	fi_connreq_t		connreq;
 	/* user must call fi_freeinfo to release info */
 	struct fi_info		*info;
 	/* connection data placed here, up to space provided */

man/fi_cm.3

@@ -93,8 +93,8 @@ waits for a connection request event.  After receiving such an event, it
 allocates a new endpoint to accept the connection.  fi_accept is invoked
 with the newly allocated endpoint passed in as the fid parameter.  If
 the listening application wishes to reject a connection request, it calls
-fi_reject with the listening endpoint passed in as the fid.  Both accept
-and reject take a reference to the connection request as an input parameter.
+fi_reject with the listening endpoint passed in as the fid.
+fi_reject takes a reference to the connection request as an input parameter.
 .PP
 A successfully accepted connection request will result in the active
 (connecting) endpoint seeing an FI_CONNECTED event on its associated

man/fi_endpoint.3

@@ -148,9 +148,8 @@ of the endpoint are defined based on the provided struct fi_info.  See
 fi_getinfo for additional details on fi_info.  fi_info flags that control
 the operation of an endpoint are defined below.
 .PP
-A single struct fi_info may only be used to allocate a single endpoint.
-If multiple endpoints are to be allocated, each should be preceded by a
-separate call to fi_getinfo.
+If an active endpoint is associated with a connection request, the
+fi_info connreq must reference the corresponding request.
 .SS "fi_close"
 Closes an endpoint and release all resources associated with it.
 .SS "fi_ep_bind"

man/fi_eq.3

@@ -214,29 +214,31 @@ Connection requests are reported using struct fi_eq_cm_entry:
 
 struct fi_eq_cm_entry {
 	fid_t            fid;        /* fid associated with request */
-	fi_connreq_t     connreq;    /* connection context */
 	struct fi_info  *info;       /* endpoint information */
 	uint8_t         data[0];     /* app connection data */
 };
 
 .fi
 Connection request events are of type FI_CONNREQ.  The fid is the passive
-endpoint.  The connreq field is used to identify the connection request
-associated with a specific event.  The value of this field must be
-passed into either fi_accept or fi_reject.
-Information regarding the requested endpoint's capabilities and
+endpoint.  Information regarding the requested endpoint's capabilities and
 attributes are available from the info field.  The application is
 responsible for freeing this structure by calling fi_freeinfo when it
-is no longer needed.
+is no longer needed.  The fi_info connreq field will reference the
+connection request associated with this event.  For an accepted
+connection, the connreq must be associated with an endpoint when
+it is opened.  Typically, this is done by simply passing the fi_info
+returned as part of the CM event into fi_endpoint().  If the connection
+is rejected, the connreq must be passed into the fi_reject call. 
 .sp
 Any application data exchanged as part of the connection
 request is placed beyond the fi_eq_cm_entry structure.  The amount of data
 available is application dependent and limited to the buffer space provided
 by the application when fi_eq_read is called.  The amount of returned data
-may be calculated using the return value to fi_eq_read.
-.sp
-The connreq field is used to identify a specific connection request.  The
-value of this field must be passed into either fi_accept or fi_reject.
+may be calculated using the return value to fi_eq_read.  Note that the amount
+of returned data is limited by the underlying connection
+protocol, and the length of any data returned may include protocol padding.
+As a result, the returned length may be larger than that specified by
+the connecting peer.
 .IP "Connection Shutdown Notification"
 Notification that a remote peer has disconnected from an active endpoint is
 done through the FI_SHUTDOWN event.  Shutdown notification uses struct

man/fi_getinfo.3

@@ -78,11 +78,10 @@ struct fi_info {
 	size_t                dest_addrlen;
 	void                  *src_addr;
 	void                  *dest_addr;
+	fi_connreq_t          connreq;
 	struct fi_ep_attr     *ep_attr;
 	struct fi_domain_attr *domain_attr;
 	struct fi_fabric_attr *fabric_attr;
-	size_t                datalen;
-	void                  *data;
 };
 .fi
 .IP "next"
@@ -120,6 +119,10 @@ is specified).
 If specified, indicates the source address.
 .IP "dest_addr"
 If specified, indicates the destination address.
+.IP "connreq"
+References a specific connection request, otherwise the field must be
+NULL.  This field is used when processing connection requests and
+responses.  See fi_eq(3), fi_reject(3), and fi_endpoint(3).
 .IP "ep_attr"
 Optionally supplied endpoint attributes.
 Endpoint attributes may be specified and returned as part of fi_getinfo.
@@ -143,14 +146,6 @@ When provided as hints, requested values of struct fi_fabric_attr should be
 set.  On output, the actual fabric attributes
 that can be provided will be returned.
 See fi_fabric(3) for details.
-.IP "datalen"
-Length of any additional data (must be specified if
-.B data
-is specified).
-.IP "data"
-Any additional data about communication interface being requested.  Data is
-provider specific.  It is primarily used as output from fi_getinfo, to be
-provided as input into fi_domain or fi_endpoint.
 .SH "ENDPOINT CAPABILITIES"
 Endpoint capabilities are obtained by OR-ing the following flags together.
 If endpoint capabilities in the hint parameter are set to 0, the underlying

prov/psm/src/psmx_init.c

@@ -213,8 +213,6 @@ static int psmx_getinfo(uint32_t version, const char *node, const char *service,
 	psmx_info->src_addr = NULL;
 	psmx_info->dest_addr = dest_addr;
 	psmx_info->fabric_attr->name = strdup("psm");
-	psmx_info->datalen = 0;
-	psmx_info->data = NULL;
 
 	*info = psmx_info;
 	return 0;