Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-5758 vos: Add garbage collection at container level #4997

Merged
merged 12 commits into from
Mar 17, 2021
Merged

DAOS-5758 vos: Add garbage collection at container level #4997

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
4130002
DAOS-5758 vos: Add garbage collection at container level
jolivier23 Mar 13, 2021
8996f13
Ok, have to make sure any keys or other entries that get
jolivier23 Mar 13, 2021
a1ffd6f
Avoid memory leak by saving offset before potentially freeing
jolivier23 Mar 13, 2021
7fef4f4
Anticipate some code review comments
jolivier23 Mar 13, 2021
27c61f9
Ensure we kick of container garbage collection upon opening
jolivier23 Mar 13, 2021
5881ab9
Disable the wait in vos_obj_delete and key delete
jolivier23 Mar 13, 2021
703205c
Need to clear the container bin so we can process it again
jolivier23 Mar 13, 2021
6b43698
Add some fault injection to test that aggregation deletions
jolivier23 Mar 13, 2021
6647f9c
Reset fail_loc in gc post
jolivier23 Mar 13, 2021
7a6748a
minor style correction
jolivier23 Mar 13, 2021
eade381
Merge branch 'master' into jvolivie/gc_bin
jolivier23 Mar 16, 2021
fb42546
Address review comments
jolivier23 Mar 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions src/vos/tests/vts_gc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,86 @@ gc_obj_test(void **state)
assert_rc_equal(rc, 0);
}

static int
gc_obj_run_destroy(struct gc_test_args *args)
{
daos_unit_oid_t *oids;
daos_handle_t coh;
daos_handle_t poh;
int i;
int rc;
uuid_t cont_id;

poh = args->gc_ctx.tsc_poh;

uuid_generate(cont_id);

rc = vos_cont_create(poh, cont_id);
if (rc) {
print_error("failed to create container: %s\n",
d_errstr(rc));
return rc;
}

gc_add_stat(STAT_CONT);
rc = vos_cont_open(poh, cont_id, &coh);
if (rc) {
print_error("failed to open container: %s\n",
d_errstr(rc));
return rc;
jolivier23 marked this conversation as resolved.
Show resolved Hide resolved
}

D_ALLOC_ARRAY(oids, obj_per_cont);
if (!oids) {
print_error("failed to allocate oids\n");
return -DER_NOMEM;
jolivier23 marked this conversation as resolved.
Show resolved Hide resolved
}

rc = gc_obj_prepare(args, coh, oids);
if (rc)
goto out;

gc_print_stat();

for (i = 0; i < obj_per_cont; i++) {
rc = vos_obj_delete(coh, oids[i]);
if (rc) {
print_error("failed to delete objects: %s\n",
d_errstr(rc));
goto out;
}
}

rc = vos_cont_close(coh);
if (rc) {
print_error("failed to close container: %s\n",
d_errstr(rc));
return rc;
}

rc = vos_cont_destroy(poh, cont_id);
if (rc) {
print_error("failed to destroy container: %s\n",
d_errstr(rc));
return rc;
}

rc = gc_wait_check(args, true);
out:
D_FREE(oids);
return rc;
}

static void
gc_obj_test_destroy(void **state)
{
struct gc_test_args *args = *state;
int rc;

rc = gc_obj_run_destroy(args);
assert_rc_equal(rc, 0);
}

static void
gc_obj_bio_test(void **state)
{
Expand Down Expand Up @@ -484,6 +564,8 @@ static const struct CMUnitTest gc_tests[] = {
gc_obj_bio_test, gc_prepare, NULL},
{ "GC04: container garbage collecting",
gc_cont_test, gc_prepare, NULL},
{ "GC05: container garbage collecting with outstanding objects",
gc_obj_test_destroy, gc_prepare, NULL},
};

int
Expand Down
8 changes: 7 additions & 1 deletion src/vos/vos_container.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ cont_df_rec_free(struct btr_instance *tins, struct btr_record *rec, void *args)
cont_df = umem_off2ptr(&tins->ti_umm, rec->rec_off);
vos_ts_evict(&cont_df->cd_ts_idx, VOS_TS_TYPE_CONT);

return gc_add_item(tins->ti_priv, GC_CONT, rec->rec_off, 0);
return gc_add_item(tins->ti_priv, DAOS_HDL_INVAL, GC_CONT, rec->rec_off,
0);
}

static int
Expand Down Expand Up @@ -100,6 +101,7 @@ cont_df_rec_alloc(struct btr_instance *tins, d_iov_t *key_iov,
}
dbtree_close(hdl);

gc_init_cont(&tins->ti_umm, cont_df);
args->ca_cont_df = cont_df;
rec->rec_off = offset;
return 0;
Expand Down Expand Up @@ -188,6 +190,9 @@ cont_free_internal(struct vos_container *cont)

dbtree_close(cont->vc_btr_hdl);

if (!d_list_empty(&cont->vc_gc_link))
d_list_del(&cont->vc_gc_link);

for (i = 0; i < VOS_IOS_CNT; i++) {
if (cont->vc_hint_ctxt[i])
vea_hint_unload(cont->vc_hint_ctxt[i]);
Expand Down Expand Up @@ -368,6 +373,7 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh)
D_INIT_LIST_HEAD(&cont->vc_dtx_committed_tmp_list);
cont->vc_dtx_committed_count = 0;
cont->vc_dtx_committed_tmp_count = 0;
D_INIT_LIST_HEAD(&cont->vc_gc_link);

/* Cache this btr object ID in container handle */
rc = dbtree_open_inplace_ex(&cont->vc_cont_df->cd_obj_root,
Expand Down
Loading