test(qp): add post send guard lifetime bounding check for extended qp

We introduce trybuild as dev-dependency, and add some misused code
which should fail to compile, so that we can check the lifetime
annotation of PostSendGuard is working as expected.

Signed-off-by: Luke Yue <lukedyue@gmail.com>

Cargo.toml

@@ -18,3 +18,6 @@ libc = "0.2"
 os_socketaddr = "0.2"
 bitmask-enum = "2.2"
 lazy_static = "1.5.0"
+
+[dev-dependencies]
+trybuild = "1.0"

tests/compiletest.rs

@@ -0,0 +1,5 @@
+#[test]
+fn compile_test() {
+    let t = trybuild::TestCases::new();
+    t.compile_fail("tests/post_send_guard/*.rs");
+}

tests/post_send_guard/one_guard_has_only_one_handle.rs

@@ -0,0 +1,96 @@
+use sideway::verbs::{
+    address::AddressHandleAttribute,
+    device,
+    device_context::Mtu,
+    queue_pair::{PostSendGuard, QueuePair, QueuePairAttribute, QueuePairState, SetInlineData, WorkRequestFlags},
+    AccessFlags,
+};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let device_list = device::DeviceList::new()?;
+    for device in &device_list {
+        let ctx = device.open().unwrap();
+
+        let pd = ctx.alloc_pd().unwrap();
+        let mr = pd.reg_managed_mr(64).unwrap();
+
+        let _comp_channel = ctx.create_comp_channel().unwrap();
+        let mut cq_builder = ctx.create_cq_builder();
+        let sq = cq_builder.setup_cqe(128).build_ex().unwrap();
+        let rq = cq_builder.setup_cqe(128).build_ex().unwrap();
+
+        let mut builder = pd.create_qp_builder();
+
+        // block for extended qp
+        {
+            let mut qp = builder
+            .setup_max_inline_data(128)
+            .setup_send_cq(&sq)
+            .setup_recv_cq(&rq)
+            .build_ex()
+            .unwrap();
+
+            println!("qp pointer is {:?}", qp);
+            // modify QP to INIT state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::Init)
+                .setup_pkey_index(0)
+                .setup_port(1)
+                .setup_access_flags(AccessFlags::LocalWrite | AccessFlags::RemoteWrite);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::Init, qp.state());
+
+            // modify QP to RTR state, set dest qp as itself
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToReceive)
+                .setup_path_mtu(Mtu::Mtu1024)
+                .setup_dest_qp_num(qp.qp_number())
+                .setup_rq_psn(1)
+                .setup_max_dest_read_atomic(0)
+                .setup_min_rnr_timer(0);
+            // setup address vector
+            let mut ah_attr = AddressHandleAttribute::new();
+            let gid_entries = ctx.query_gid_table().unwrap();
+
+            ah_attr
+                .setup_dest_lid(1)
+                .setup_port(1)
+                .setup_service_level(1)
+                .setup_grh_src_gid_index(gid_entries[0].gid_index().try_into().unwrap())
+                .setup_grh_dest_gid(&gid_entries[0].gid())
+                .setup_grh_hop_limit(64);
+            attr.setup_address_vector(&ah_attr);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToReceive, qp.state());
+
+            // modify QP to RTS state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToSend)
+                .setup_sq_psn(1)
+                .setup_timeout(12)
+                .setup_retry_cnt(7)
+                .setup_rnr_retry(7)
+                .setup_max_read_atomic(0);
+
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToSend, qp.state());
+
+            let mut guard = qp.start_post_send().unwrap();
+            let buf = vec![0, 1, 2, 3];
+
+            let write_handle = guard
+                .construct_wr(233, WorkRequestFlags::Signaled | WorkRequestFlags::Inline)
+                .setup_write(mr.rkey(), mr.buf.data.as_ptr() as _);
+
+            // while holding a write handle, we can't build a send handle at the same time
+            let _send_handle = guard.construct_wr(2, 0.into()).setup_send();
+
+            write_handle.setup_inline_data(&buf);
+        }
+    }
+
+    Ok(())
+}

tests/post_send_guard/one_guard_has_only_one_handle.stderr

@@ -0,0 +1,11 @@
+error[E0499]: cannot borrow `guard` as mutable more than once at a time
+  --> tests/post_send_guard/one_guard_has_only_one_handle.rs:89:32
+   |
+84 |             let write_handle = guard
+   |                                ----- first mutable borrow occurs here
+...
+89 |             let _send_handle = guard.construct_wr(2, 0.into()).setup_send();
+   |                                ^^^^^ second mutable borrow occurs here
+90 |
+91 |             write_handle.setup_inline_data(&buf);
+   |             ------------ first borrow later used here

tests/post_send_guard/one_guard_has_only_one_wr.rs

@@ -0,0 +1,93 @@
+use sideway::verbs::{
+    address::AddressHandleAttribute,
+    device,
+    device_context::Mtu,
+    queue_pair::{PostSendGuard, QueuePair, QueuePairAttribute, QueuePairState, WorkRequestFlags},
+    AccessFlags,
+};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let device_list = device::DeviceList::new()?;
+    for device in &device_list {
+        let ctx = device.open().unwrap();
+
+        let pd = ctx.alloc_pd().unwrap();
+        let mr = pd.reg_managed_mr(64).unwrap();
+
+        let _comp_channel = ctx.create_comp_channel().unwrap();
+        let mut cq_builder = ctx.create_cq_builder();
+        let sq = cq_builder.setup_cqe(128).build_ex().unwrap();
+        let rq = cq_builder.setup_cqe(128).build_ex().unwrap();
+
+        let mut builder = pd.create_qp_builder();
+
+        // block for extended qp
+        {
+            let mut qp = builder
+                .setup_max_inline_data(128)
+                .setup_send_cq(&sq)
+                .setup_recv_cq(&rq)
+                .build_ex()
+                .unwrap();
+
+            println!("qp pointer is {:?}", qp);
+            // modify QP to INIT state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::Init)
+                .setup_pkey_index(0)
+                .setup_port(1)
+                .setup_access_flags(AccessFlags::LocalWrite | AccessFlags::RemoteWrite);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::Init, qp.state());
+
+            // modify QP to RTR state, set dest qp as itself
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToReceive)
+                .setup_path_mtu(Mtu::Mtu1024)
+                .setup_dest_qp_num(qp.qp_number())
+                .setup_rq_psn(1)
+                .setup_max_dest_read_atomic(0)
+                .setup_min_rnr_timer(0);
+            // setup address vector
+            let mut ah_attr = AddressHandleAttribute::new();
+            let gid_entries = ctx.query_gid_table().unwrap();
+
+            ah_attr
+                .setup_dest_lid(1)
+                .setup_port(1)
+                .setup_service_level(1)
+                .setup_grh_src_gid_index(gid_entries[0].gid_index().try_into().unwrap())
+                .setup_grh_dest_gid(&gid_entries[0].gid())
+                .setup_grh_hop_limit(64);
+            attr.setup_address_vector(&ah_attr);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToReceive, qp.state());
+
+            // modify QP to RTS state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToSend)
+                .setup_sq_psn(1)
+                .setup_timeout(12)
+                .setup_retry_cnt(7)
+                .setup_rnr_retry(7)
+                .setup_max_read_atomic(0);
+
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToSend, qp.state());
+
+            let mut guard = qp.start_post_send().unwrap();
+
+            let wr = guard.construct_wr(233, WorkRequestFlags::Signaled | WorkRequestFlags::Inline);
+
+            // while holding a write handle, we can't build a send handle at the same time
+            let _wr_2 = guard.construct_wr(2, 0.into());
+
+            let _write_handle = wr.setup_write(mr.rkey(), mr.buf.data.as_ptr() as _);
+        }
+    }
+
+    Ok(())
+}

tests/post_send_guard/one_guard_has_only_one_wr.stderr

@@ -0,0 +1,11 @@
+error[E0499]: cannot borrow `guard` as mutable more than once at a time
+  --> tests/post_send_guard/one_guard_has_only_one_wr.rs:86:25
+   |
+83 |             let wr = guard.construct_wr(233, WorkRequestFlags::Signaled | WorkRequestFlags::Inline);
+   |                      ----- first mutable borrow occurs here
+...
+86 |             let _wr_2 = guard.construct_wr(2, 0.into());
+   |                         ^^^^^ second mutable borrow occurs here
+87 |
+88 |             let _write_handle = wr.setup_write(mr.rkey(), mr.buf.data.as_ptr() as _);
+   |                                 -- first borrow later used here

tests/post_send_guard/one_qp_has_only_one_guard.rs

@@ -0,0 +1,91 @@
+use sideway::verbs::{
+    address::AddressHandleAttribute,
+    device,
+    device_context::Mtu,
+    queue_pair::{PostSendGuard, QueuePair, QueuePairAttribute, QueuePairState},
+    AccessFlags,
+};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let device_list = device::DeviceList::new()?;
+    for device in &device_list {
+        let ctx = device.open().unwrap();
+
+        let pd = ctx.alloc_pd().unwrap();
+        let _mr = pd.reg_managed_mr(64).unwrap();
+
+        let _comp_channel = ctx.create_comp_channel().unwrap();
+        let mut cq_builder = ctx.create_cq_builder();
+        let sq = cq_builder.setup_cqe(128).build_ex().unwrap();
+        let rq = cq_builder.setup_cqe(128).build_ex().unwrap();
+
+        let mut builder = pd.create_qp_builder();
+
+        // block for extended qp
+        {
+            let mut qp = builder
+                .setup_max_inline_data(128)
+                .setup_send_cq(&sq)
+                .setup_recv_cq(&rq)
+                .build_ex()
+                .unwrap();
+
+            println!("qp pointer is {:?}", qp);
+            // modify QP to INIT state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::Init)
+                .setup_pkey_index(0)
+                .setup_port(1)
+                .setup_access_flags(AccessFlags::LocalWrite | AccessFlags::RemoteWrite);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::Init, qp.state());
+
+            // modify QP to RTR state, set dest qp as itself
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToReceive)
+                .setup_path_mtu(Mtu::Mtu1024)
+                .setup_dest_qp_num(qp.qp_number())
+                .setup_rq_psn(1)
+                .setup_max_dest_read_atomic(0)
+                .setup_min_rnr_timer(0);
+            // setup address vector
+            let mut ah_attr = AddressHandleAttribute::new();
+            let gid_entries = ctx.query_gid_table().unwrap();
+
+            ah_attr
+                .setup_dest_lid(1)
+                .setup_port(1)
+                .setup_service_level(1)
+                .setup_grh_src_gid_index(gid_entries[0].gid_index().try_into().unwrap())
+                .setup_grh_dest_gid(&gid_entries[0].gid())
+                .setup_grh_hop_limit(64);
+            attr.setup_address_vector(&ah_attr);
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToReceive, qp.state());
+
+            // modify QP to RTS state
+            let mut attr = QueuePairAttribute::new();
+            attr.setup_state(QueuePairState::ReadyToSend)
+                .setup_sq_psn(1)
+                .setup_timeout(12)
+                .setup_retry_cnt(7)
+                .setup_rnr_retry(7)
+                .setup_max_read_atomic(0);
+
+            qp.modify(&attr).unwrap();
+
+            assert_eq!(QueuePairState::ReadyToSend, qp.state());
+
+            let guard = qp.start_post_send().unwrap();
+
+            // while holding a post send guard, we can't build a post send guard at the same time
+            let _guard_2 = qp.start_post_send().unwrap();
+
+            let _res = guard.post().unwrap();
+        }
+    }
+
+    Ok(())
+}

tests/post_send_guard/one_qp_has_only_one_guard.stderr

@@ -0,0 +1,11 @@
+error[E0499]: cannot borrow `qp` as mutable more than once at a time
+  --> tests/post_send_guard/one_qp_has_only_one_guard.rs:84:28
+   |
+81 |             let guard = qp.start_post_send().unwrap();
+   |                         -- first mutable borrow occurs here
+...
+84 |             let _guard_2 = qp.start_post_send().unwrap();
+   |                            ^^ second mutable borrow occurs here
+85 |
+86 |             let _res = guard.post().unwrap();
+   |                        ----- first borrow later used here