Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-6999 tests: add tests to verify reintegrate inflight IO (#5058) #5115

Merged
merged 1 commit into from
Mar 26, 2021
Merged

DAOS-6999 tests: add tests to verify reintegrate inflight IO (#5058) #5115

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 71 additions & 10 deletions src/tests/suite/daos_rebuild_simple.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,67 @@
#define REBUILD_POOL_SIZE (4ULL << 30)

#define DATA_SIZE (1048576 * 2 + 512)

static int
reintegrate_inflight_io(void *data)
{
test_arg_t *arg = data;
daos_obj_id_t oid = *(daos_obj_id_t *)arg->rebuild_cb_arg;
struct ioreq req;
int i;

rebuild_pool_connect_internal(arg);
ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg);
for (i = 0; i < 5; i++) {
char key[32];
daos_recx_t recx;
char buf[DATA_SIZE];

sprintf(key, "d_inflight_%d", i);
insert_single(key, "a_key", 0, "data", strlen("data") + 1,
DAOS_TX_NONE, &req);

sprintf(key, "d_inflight_1M_%d", i);
recx.rx_idx = 0;
recx.rx_nr = DATA_SIZE;
memset(buf, 'a', DATA_SIZE);
insert_recxs(key, "a_key_1M", 1, DAOS_TX_NONE, &recx, 1,
buf, DATA_SIZE, &req);
}
ioreq_fini(&req);
rebuild_pool_disconnect_internal(arg);
if (arg->myrank == 0)
daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0,
NULL);
return 0;
}

static void
reintegrate_with_inflight_io(test_arg_t *arg, d_rank_t rank, int tgt)
{
daos_obj_id_t oid;
int rc;

oid = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0,
0, arg->myrank);

oid = dts_oid_set_rank(oid, rank);
arg->rebuild_cb = reintegrate_inflight_io;
arg->rebuild_cb_arg = &oid;

/* To make sure the IO will be done before reintegration is done */
if (arg->myrank == 0)
daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC,
DAOS_REBUILD_TGT_REBUILD_HANG, 0, NULL);
reintegrate_single_pool_target(arg, rank, tgt);
arg->rebuild_cb = NULL;
arg->rebuild_cb_arg = NULL;

rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
}

static void
rebuild_dkeys(void **state)
{
Expand Down Expand Up @@ -73,7 +134,8 @@ rebuild_dkeys(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, kill_rank, -1);
reintegrate_with_inflight_io(arg, kill_rank, -1);

rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -127,8 +189,7 @@ rebuild_akeys(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);


reintegrate_single_pool_target(arg, kill_rank, tgt);
reintegrate_with_inflight_io(arg, kill_rank, tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -172,7 +233,7 @@ rebuild_indexes(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -231,7 +292,7 @@ rebuild_snap_update_recs(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < SNAP_CNT; i++) {
rc = daos_obj_verify(arg->coh, oid, snap_epoch[i]);
if (rc != 0)
Expand Down Expand Up @@ -290,7 +351,7 @@ rebuild_snap_punch_recs(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < SNAP_CNT; i++) {
rc = daos_obj_verify(arg->coh, oid, snap_epoch[i]);
if (rc != 0)
Expand Down Expand Up @@ -377,7 +438,7 @@ rebuild_snap_update_keys(void **state)
assert_int_equal(number, SNAP_CNT);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -479,7 +540,7 @@ rebuild_snap_punch_keys(void **state)
assert_int_equal(number, SNAP_CNT);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -655,7 +716,7 @@ rebuild_large_rec(void **state)
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX);
if (rc != 0)
assert_rc_equal(rc, -DER_NOSYS);
Expand Down Expand Up @@ -690,7 +751,7 @@ rebuild_objects(void **state)
assert_rc_equal(rc, -DER_NOSYS);
}

reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt);
reintegrate_with_inflight_io(arg, ranks_to_kill[0], tgt);
for (i = 0; i < OBJ_NR; i++) {
rc = daos_obj_verify(arg->coh, oids[i], DAOS_EPOCH_MAX);
if (rc != 0)
Expand Down