Atomic write with IBV_SEND_FENCE flag

pmem · Nov 26, 2020 · 0da3f89 · 0da3f89
1 parent bd17e8f
commit 0da3f89
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 8 deletions.
diff --git a/src/conn.c b/src/conn.c
@@ -322,7 +322,7 @@ rpma_write(struct rpma_conn *conn,
  return rpma_mr_write(conn->id->qp,
  dst, dst_offset,
  src, src_offset,
- len, flags, op_context);
+ len, flags, op_context, false);
 }
 
 /*
@@ -343,7 +343,7 @@ rpma_write_atomic(struct rpma_conn *conn,
  return rpma_mr_write(conn->id->qp,
  dst, dst_offset,
  src, src_offset,
- RPMA_ATOMIC_WRITE_ALIGNMENT, flags, op_context);
+ RPMA_ATOMIC_WRITE_ALIGNMENT, flags, op_context, true);
 }
 
 /*

diff --git a/src/mr.c b/src/mr.c
@@ -147,7 +147,7 @@ int
 rpma_mr_write(struct ibv_qp *qp,
  struct rpma_mr_remote *dst, size_t dst_offset,
  const struct rpma_mr_local *src, size_t src_offset,
- size_t len, int flags, const void *op_context)
+ size_t len, int flags, const void *op_context, bool fence)
 {
  struct ibv_send_wr wr;
  struct ibv_sge sge;
@@ -168,6 +168,8 @@ rpma_mr_write(struct ibv_qp *qp,
  wr.opcode = IBV_WR_RDMA_WRITE;
  wr.send_flags = (flags & RPMA_F_COMPLETION_ON_SUCCESS) ?
  IBV_SEND_SIGNALED : 0;
+ wr.send_flags |= (fence == true) ?
+ IBV_SEND_FENCE : 0;
 
  struct ibv_send_wr *bad_wr;
  int ret = ibv_post_send(qp, &wr, &bad_wr);

diff --git a/src/mr.h b/src/mr.h
@@ -38,7 +38,7 @@ int rpma_mr_read(struct ibv_qp *qp,
 int rpma_mr_write(struct ibv_qp *qp,
  struct rpma_mr_remote *dst, size_t dst_offset,
  const struct rpma_mr_local *src, size_t src_offset,
- size_t len, int flags, const void *op_context);
+ size_t len, int flags, const void *op_context, bool fence);
 
 /*
  * ASSUMPTIONS

diff --git a/tests/unit/common/mocks-rpma-mr.c b/tests/unit/common/mocks-rpma-mr.c
@@ -45,7 +45,7 @@ int
 rpma_mr_write(struct ibv_qp *qp,
  struct rpma_mr_remote *dst, size_t dst_offset,
  const struct rpma_mr_local *src, size_t src_offset,
- size_t len, int flags, const void *op_context)
+ size_t len, int flags, const void *op_context, bool fence)
 {
  assert_non_null(qp);
  assert_non_null(dst);
@@ -60,6 +60,7 @@ rpma_mr_write(struct ibv_qp *qp,
  check_expected(len);
  check_expected(flags);
  check_expected_ptr(op_context);
+ check_expected(fence);
 
  return mock_type(int);
 }

diff --git a/tests/unit/common/test-common.h b/tests/unit/common/test-common.h
@@ -29,6 +29,8 @@
 #define MOCK_LEN (size_t)0xC415
 #define MOCK_FLAGS (int)0xC416
 #define MOCK_OP_CONTEXT (void *)0xC417
+#define MOCK_NOFENCE false
+#define MOCK_FENCE true
 
 #define MOCK_OK 0
 #define MOCK_ERRNO 123456

diff --git a/tests/unit/conn/conn-write.c b/tests/unit/conn/conn-write.c
@@ -105,6 +105,7 @@ write__success(void **cstate_ptr)
  expect_value(rpma_mr_write, len, MOCK_LEN);
  expect_value(rpma_mr_write, flags, MOCK_FLAGS);
  expect_value(rpma_mr_write, op_context, MOCK_OP_CONTEXT);
+ expect_value(rpma_mr_write, fence, MOCK_NOFENCE);
  will_return(rpma_mr_write, MOCK_OK);
 
  /* run test */

diff --git a/tests/unit/conn/conn-write_atomic.c b/tests/unit/conn/conn-write_atomic.c
@@ -120,6 +120,7 @@ write_atomic__success(void **cstate_ptr)
  expect_value(rpma_mr_write, len, RPMA_ATOMIC_WRITE_ALIGNMENT);
  expect_value(rpma_mr_write, flags, MOCK_FLAGS);
  expect_value(rpma_mr_write, op_context, MOCK_OP_CONTEXT);
+ expect_value(rpma_mr_write, fence, MOCK_FENCE);
  will_return(rpma_mr_write, MOCK_OK);
 
  /* run test */

diff --git a/tests/unit/mr/mr-write.c b/tests/unit/mr/mr-write.c
@@ -38,7 +38,7 @@ write__COMPL_ALWAYS_failed_E_PROVIDER(void **mrs_ptr)
  int ret = rpma_mr_write(MOCK_QP, mrs->remote, MOCK_DST_OFFSET,
  mrs->local, MOCK_SRC_OFFSET,
  MOCK_LEN, RPMA_F_COMPLETION_ALWAYS,
- MOCK_OP_CONTEXT);
+ MOCK_OP_CONTEXT, MOCK_NOFENCE);
 
  /* verify the results */
  assert_int_equal(ret, RPMA_E_PROVIDER);
@@ -66,7 +66,7 @@ write__COMPL_ON_ERROR_failed_E_PROVIDER(void **mrs_ptr)
  int ret = rpma_mr_write(MOCK_QP, mrs->remote, MOCK_DST_OFFSET,
  mrs->local, MOCK_SRC_OFFSET,
  MOCK_LEN, RPMA_F_COMPLETION_ON_ERROR,
- MOCK_OP_CONTEXT);
+ MOCK_OP_CONTEXT, MOCK_NOFENCE);
 
  /* verify the results */
  assert_int_equal(ret, RPMA_E_PROVIDER);
@@ -93,7 +93,7 @@ write__success(void **mrs_ptr)
  int ret = rpma_mr_write(MOCK_QP, mrs->remote, MOCK_DST_OFFSET,
  mrs->local, MOCK_SRC_OFFSET,
  MOCK_LEN, RPMA_F_COMPLETION_ALWAYS,
- MOCK_OP_CONTEXT);
+ MOCK_OP_CONTEXT, MOCK_NOFENCE);
 
  /* verify the results */
  assert_int_equal(ret, MOCK_OK);