Skip to content

Commit

Permalink
smc: remote memory buffers (RMBs)
Browse files Browse the repository at this point in the history
* allocate data RMB memory for sending and receiving
* size depends on the maximum socket send and receive buffers
* allocated RMBs are kept during life time of the owning link group
* map the allocated RMBs to DMA

Signed-off-by: Ursula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Ursula Braun authored and davem330 committed Jan 9, 2017
1 parent 0cfdd8f commit cd6851f
Show file tree
Hide file tree
Showing 7 changed files with 342 additions and 7 deletions.
29 changes: 26 additions & 3 deletions net/smc/af_smc.c
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ static void smc_conn_save_peer_info(struct smc_sock *smc,
struct smc_clc_msg_accept_confirm *clc)
{
smc->conn.peer_conn_idx = clc->conn_idx;
smc->conn.peer_rmbe_size = smc_uncompress_bufsize(clc->rmbe_size);
atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
}

static void smc_link_save_peer_info(struct smc_link *link,
Expand Down Expand Up @@ -323,6 +325,18 @@ static int smc_connect_rdma(struct smc_sock *smc)
link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];

smc_conn_save_peer_info(smc, &aclc);

rc = smc_sndbuf_create(smc);
if (rc) {
reason_code = SMC_CLC_DECL_MEM;
goto decline_rdma_unlock;
}
rc = smc_rmb_create(smc);
if (rc) {
reason_code = SMC_CLC_DECL_MEM;
goto decline_rdma_unlock;
}

if (local_contact == SMC_FIRST_CONTACT)
smc_link_save_peer_info(link, &aclc);
/* tbd in follow-on patch: more steps to setup RDMA communcication,
Expand Down Expand Up @@ -598,9 +612,16 @@ static void smc_listen_work(struct work_struct *work)
}
link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];

/* tbd in follow-on patch: more steps to setup RDMA communcication,
* create rmbs, map rmbs
*/
rc = smc_sndbuf_create(new_smc);
if (rc) {
reason_code = SMC_CLC_DECL_MEM;
goto decline_rdma;
}
rc = smc_rmb_create(new_smc);
if (rc) {
reason_code = SMC_CLC_DECL_MEM;
goto decline_rdma;
}

rc = smc_clc_send_accept(new_smc, local_contact);
if (rc)
Expand Down Expand Up @@ -1047,6 +1068,8 @@ static int smc_create(struct net *net, struct socket *sock, int protocol,
IPPROTO_TCP, &smc->clcsock);
if (rc)
sk_common_release(sk);
smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);

out:
return rc;
Expand Down
45 changes: 45 additions & 0 deletions net/smc/smc.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,16 @@ struct smc_connection {
struct smc_link_group *lgr; /* link group of connection */
u32 alert_token_local; /* unique conn. id */
u8 peer_conn_idx; /* from tcp handshake */
int peer_rmbe_size; /* size of peer rx buffer */
atomic_t peer_rmbe_space;/* remaining free bytes in peer
* rmbe
*/

struct smc_buf_desc *sndbuf_desc; /* send buffer descriptor */
int sndbuf_size; /* sndbuf size <== sock wmem */
struct smc_buf_desc *rmb_desc; /* RMBE descriptor */
int rmbe_size; /* RMBE size <== sock rmem */
int rmbe_size_short;/* compressed notation */
};

struct smc_sock { /* smc sock container */
Expand Down Expand Up @@ -76,6 +86,41 @@ static inline u32 ntoh24(u8 *net)
return be32_to_cpu(t);
}

#define SMC_BUF_MIN_SIZE 16384 /* minimum size of an RMB */

#define SMC_RMBE_SIZES 16 /* number of distinct sizes for an RMBE */
/* theoretically, the RFC states that largest size would be 512K,
* i.e. compressed 5 and thus 6 sizes (0..5), despite
* struct smc_clc_msg_accept_confirm.rmbe_size being a 4 bit value (0..15)
*/

/* convert the RMB size into the compressed notation - minimum 16K.
* In contrast to plain ilog2, this rounds towards the next power of 2,
* so the socket application gets at least its desired sndbuf / rcvbuf size.
*/
static inline u8 smc_compress_bufsize(int size)
{
u8 compressed;

if (size <= SMC_BUF_MIN_SIZE)
return 0;

size = (size - 1) >> 14;
compressed = ilog2(size) + 1;
if (compressed >= SMC_RMBE_SIZES)
compressed = SMC_RMBE_SIZES - 1;
return compressed;
}

/* convert the RMB size from compressed notation into integer */
static inline int smc_uncompress_bufsize(u8 compressed)
{
u32 size;

size = 0x00000001 << (((int)compressed) + 14);
return (int)size;
}

#ifdef CONFIG_XFRM
static inline bool using_ipsec(struct smc_sock *smc)
{
Expand Down
6 changes: 3 additions & 3 deletions net/smc/smc_clc.c
Original file line number Diff line number Diff line change
Expand Up @@ -252,13 +252,13 @@ int smc_clc_send_accept(struct smc_sock *new_smc, int srv_first_contact)
SMC_GID_SIZE);
memcpy(&aclc.lcl.mac, link->smcibdev->mac[link->ibport - 1],
sizeof(link->smcibdev->mac[link->ibport - 1]));

/* tbd in follow-on patch: fill in rmb-related values */

hton24(aclc.qpn, link->roce_qp->qp_num);
aclc.conn_idx = 1; /* as long as 1 RMB = 1 RMBE */
aclc.rmbe_alert_token = htonl(conn->alert_token_local);
aclc.qp_mtu = link->path_mtu;
aclc.rmbe_size = conn->rmbe_size_short,
aclc.rmb_dma_addr =
cpu_to_be64((u64)conn->rmb_desc->dma_addr[SMC_SINGLE_LINK]);
hton24(aclc.psn, link->psn_initial);
memcpy(aclc.trl.eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER));

Expand Down
Loading

0 comments on commit cd6851f

Please sign in to comment.