DAOS-7165 VOS: free VOS pools attached on GC (#5472)

src/include/daos_srv/vos.h

@@ -1052,8 +1052,8 @@ int
 vos_pool_ctl(daos_handle_t poh, enum vos_pool_opc opc);
 
 int
-vos_gc_pool_run(daos_handle_t poh, int credits,
-		bool (*yield_func)(void *arg), void *yield_arg);
+vos_gc_pool(daos_handle_t poh, int credits, bool (*yield_func)(void *arg),
+	    void *yield_arg);
 bool
 vos_gc_pool_idle(daos_handle_t poh);
 

src/pool/srv_target.c

@@ -87,8 +87,8 @@ gc_ult(void *arg)
 
 	D_ASSERT(child->spc_gc_req != NULL);
 	while (!dss_ult_exiting(child->spc_gc_req)) {
-		rc = vos_gc_pool_run(child->spc_hdl, -1, dss_ult_yield,
-				     (void *)child->spc_gc_req);
+		rc = vos_gc_pool(child->spc_hdl, -1, dss_ult_yield,
+				 (void *)child->spc_gc_req);
 		if (rc < 0)
 			D_ERROR(DF_UUID"[%d]: GC pool run failed. "DF_RC"\n",
 				DP_UUID(child->spc_uuid), dmi->dmi_tgt_id,

src/rdb/rdb_raft.c

@@ -1490,7 +1490,7 @@ rdb_compactd(void *arg)
 				": %d\n", DP_DB(db), base, rc);
 			break;
 		}
-		vos_gc_pool_run(db->d_pool, -1, rdb_gc_yield, NULL);
+		vos_gc_pool(db->d_pool, -1, rdb_gc_yield, NULL);
 	}
 	D_DEBUG(DB_MD, DF_DB": compactd stopping\n", DP_DB(db));
 }

src/vos/sys_db.c

@@ -252,7 +252,7 @@ db_delete(struct sys_db *db, char *table, d_iov_t *key)
 	if (rc == 0) {
 		int creds = 100;
 		/* vos_obj_del_key() wouldn't free space */
-		vos_gc_pool(vdb->db_poh, &creds);
+		vos_gc_pool_tight(vdb->db_poh, &creds);
 	}
 	return rc;
 }

src/vos/tests/vts_gc.c

@@ -198,7 +198,7 @@ gc_wait_check(struct gc_test_args *args, bool cont_delete)
 	while (1) {
 		int	creds = 64;
 
-		rc = vos_gc_pool(args->gc_ctx.tsc_poh, &creds);
+		rc = vos_gc_pool_tight(args->gc_ctx.tsc_poh, &creds);
 		if (rc) {
 			print_error("gc pool failed: %s\n", d_errstr(rc));
 			return rc;

src/vos/vos_common.c

@@ -325,7 +325,21 @@ vos_tls_fini(void *data)
 {
 	struct vos_tls *tls = data;
 
-	D_ASSERT(d_list_empty(&tls->vtl_gc_pools));
+	/* All GC callers should have exited, but they can still leave
+	 * uncleaned pools behind. It is OK to free these pool handles with
+	 * leftover, because GC can clean up leftover when it starts again.
+	 */
+	D_ASSERTF(tls->vtl_gc_running == 0, "GC running = %d\n",
+		  tls->vtl_gc_running);
+
+	while (!d_list_empty(&tls->vtl_gc_pools)) {
+		struct vos_pool *pool;
+
+		pool = d_list_entry(tls->vtl_gc_pools.next,
+				    struct vos_pool, vp_gc_link);
+		gc_del_pool(pool);
+	}
+
 	if (tls->vtl_ocache)
 		vos_obj_cache_destroy(tls->vtl_ocache);
 

src/vos/vos_gc.c

@@ -1030,9 +1030,8 @@ gc_wait(void)
 #endif
 }
 
-/** public API to reclaim space for a opened pool */
 int
-vos_gc_pool(daos_handle_t poh, int *credits)
+vos_gc_pool_tight(daos_handle_t poh, int *credits)
 {
 	struct vos_pool *pool = vos_hdl2pool(poh);
 	bool		 empty;
@@ -1080,17 +1079,20 @@ vos_gc_yield(bool (*yield_func)(void *arg), void *yield_arg)
 	return false;
 }
 
+/** public API to reclaim space for a opened pool */
 int
-vos_gc_pool_run(daos_handle_t poh, int credits,
-		bool (*yield_func)(void *arg), void *yield_arg)
+vos_gc_pool(daos_handle_t poh, int credits, bool (*yield_func)(void *arg),
+	    void *yield_arg)
 {
 	struct vos_pool	*pool = vos_hdl2pool(poh);
+	struct vos_tls	*tls  = vos_tls_get();
 	int		 rc, total = 0;
 
 	D_ASSERT(daos_handle_is_valid(poh));
 	if (!gc_have_pool(pool))
 		return 0; /* nothing to reclaim for this pool */
 
+	tls->vtl_gc_running++;
 	/*
 	 * Pause flushing free extents in VEA aging buffer, otherwise,
 	 * there'll be way more fragments to be processed.
@@ -1104,7 +1106,7 @@ vos_gc_pool_run(daos_handle_t poh, int credits,
 			creds = credits - total;
 
 		total += creds;
-		rc = vos_gc_pool(poh, &creds);
+		rc = vos_gc_pool_tight(poh, &creds);
 		if (rc) {
 			D_ERROR("GC pool failed: %s\n", d_errstr(rc));
 			break;
@@ -1128,6 +1130,9 @@ vos_gc_pool_run(daos_handle_t poh, int credits,
 
 	if (total != 0) /* did something */
 		D_DEBUG(DB_TRACE, "GC consumed %d credits\n", total);
+
+	D_ASSERT(tls->vtl_gc_running > 0);
+	tls->vtl_gc_running--;
 	return rc;
 }
 

src/vos/vos_internal.h

@@ -1093,7 +1093,7 @@ int
 gc_add_item(struct vos_pool *pool, daos_handle_t coh,
 	    enum vos_gc_type type, umem_off_t item_off, uint64_t args);
 int
-vos_gc_pool(daos_handle_t poh, int *credits);
+vos_gc_pool_tight(daos_handle_t poh, int *credits);
 void
 gc_reserve_space(daos_size_t *rsrvd);
 

src/vos/vos_tls.h

@@ -30,6 +30,8 @@ struct dtx_handle;
 struct vos_tls {
 	/** pools registered for GC */
 	d_list_t			 vtl_gc_pools;
+	/** tracking GC running status */
+	int				 vtl_gc_running;
 	/* PMDK transaction stage callback data */
 	struct umem_tx_stage_data	 vtl_txd;
 	/** XXX: The DTX handle.