Skip to content

Commit

Permalink
DAOS-6999 tests: add tests to verify reintegrate inflight IO (#5058)
Browse files Browse the repository at this point in the history
  • Loading branch information
wangdi committed Mar 24, 2021
1 parent 937582b commit 63e81a9
Showing 1 changed file with 71 additions and 10 deletions.
81 changes: 71 additions & 10 deletions src/tests/suite/daos_rebuild_simple.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,67 @@
#define REBUILD_POOL_SIZE (4ULL << 30)

#define DATA_SIZE (1048576 * 2 + 512)

static int
reintegrate_inflight_io(void *data)
{
test_arg_t *arg = data;
daos_obj_id_t oid = *(daos_obj_id_t *)arg->rebuild_cb_arg;
struct ioreq req;
int i;

rebuild_pool_connect_internal(arg);
ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg);
for (i = 0; i < 5; i++) {
char key[32];
daos_recx_t recx;
char buf[DATA_SIZE];

sprintf(key, "d_inflight_%d", i);
insert_single(key, "a_key", 0, "data", strlen("data") + 1,
DAOS_TX_NONE, &req);

sprintf(key, "d_inflight_1M_%d", i);
recx.rx_idx = 0;
recx.rx_nr = DATA_SIZE;
memset(buf, 'a', DATA_SIZE);
insert_recxs(key, "a_key_1M", 1, DAOS_TX_NONE, &recx, 1,
buf, DATA_SIZE, &req);
}
ioreq_fini(&req);
rebuild_pool_disconnect_internal(arg);
if (arg->myrank == 0)
daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0,
NULL);
return 0;
}

static void
reintegrate_with_inflight_io(test_arg_t *arg, d_rank_t rank, int tgt)
{
daos_obj_id_t oid;
int rc;

oid = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0,
0, arg->myrank);

oid = dts_oid_set_rank(oid, rank);
arg->rebuild_cb = reintegrate_inflight_io;
arg->rebuild_cb_arg = &oid;

/* To make sure the IO will be done before reintegration is done */
if (arg->myrank == 0)
daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC,
DAOS_REBUILD_TGT_REBUILD_HANG, 0, NULL);
reintegrate_single_pool_target(arg, rank, tgt);
arg->rebuild_cb = NULL;
arg->rebuild_cb_arg = NULL;

rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
}

static void
rebuild_dkeys(void **state)
{
Expand Down Expand Up @@ -73,7 +134,8 @@ rebuild_dkeys(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, kill_rank, -1);
reintegrate_with_inflight_io(arg, kill_rank, -1);

rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -127,8 +189,7 @@ rebuild_akeys(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);


reintegrate_single_pool_target(arg, kill_rank, tgt);
reintegrate_with_inflight_io(arg, kill_rank, tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -172,7 +233,7 @@ rebuild_indexes(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -231,7 +292,7 @@ rebuild_snap_update_recs(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < SNAP_CNT; i++) {
rc = daos_obj_verify(arg->coh, oid, snap_epoch[i]);
if (rc != 0)
Expand Down Expand Up @@ -290,7 +351,7 @@ rebuild_snap_punch_recs(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < SNAP_CNT; i++) {
rc = daos_obj_verify(arg->coh, oid, snap_epoch[i]);
if (rc != 0)
Expand Down Expand Up @@ -377,7 +438,7 @@ rebuild_snap_update_keys(void **state)
assert_int_equal(number, SNAP_CNT);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -479,7 +540,7 @@ rebuild_snap_punch_keys(void **state)
assert_int_equal(number, SNAP_CNT);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -655,7 +716,7 @@ rebuild_large_rec(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -690,7 +751,7 @@ rebuild_objects(void **state)
assert_rc_equal(rc, -DER_NOSYS);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < OBJ_NR; i++) {
rc = daos_obj_verify(arg->coh, oids[i], DAOS_EPOCH_MAX);
if (rc != 0)
Expand Down

0 comments on commit 63e81a9

Please sign in to comment.