Skip to content

Commit

Permalink
Added comments for GC
Browse files Browse the repository at this point in the history
  • Loading branch information
agarwal-navin committed Jan 12, 2021
1 parent ae2a650 commit 7ef8fce
Show file tree
Hide file tree
Showing 9 changed files with 112 additions and 19 deletions.
31 changes: 28 additions & 3 deletions packages/runtime/container-runtime/src/dataStoreContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -428,10 +428,21 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
return { ...summarizeResult, id: this.id };
}

/**
* Returns the data used for garbage collection. This includes a list of GC nodes that represent this data store
* including any of its child channel contexts. Each node has a set of outbound routes to other GC nodes in the
* document.
* If there is no new data in this data store since the last summary, previous GC data is used.
* If there is new data, the GC data is generated again (by calling getGCDataInternal).
*/
public async getGCData(): Promise<IGarbageCollectionData> {
return this.summarizerNode.getGCData();
}

/**
* Generates data used for garbage collection. This is called when there is new data since last summary. It
* realizes the data store and calls into each channel context to get its GC data.
*/
private async getGCDataInternal(): Promise<IGarbageCollectionData> {
await this.realize();
assert(this.channel !== undefined, "Channel should not be undefined when running GC");
Expand All @@ -446,7 +457,12 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
}

/**
* After GC has run, called to notify this data store of routes that are used in it.
* After GC has run, called to notify the data store of routes used in it. These are used for the following:
* 1. To identify if this data store is being referenced in the document or not.
* 2. To determine if it needs to re-summarize in case used routes changed since last summary.
* 3. These are added to the summary generated by the data store.
* 4. To notify child contexts of their used routes. This is done immediately if the data store is loaded. Else,
* it is done when realizing the data store.
* @param usedRoutes - The routes that are used in this data store.
*/
public updateUsedRoutes(usedRoutes: string[]) {
Expand All @@ -464,7 +480,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
}

/**
* Calls the channel to update used routes of its child contexts.
* Updates the used routes of the channel and its child contexts. The channel must be loaded before calling this.
* It is called in these two scenarions:
* 1. When the used routes of the data store is updated and the data store is loaded.
* 2. When the data store is realized. This updates the channel's used routes as per last GC run.
*/
private updateChannelUsedRoutes() {
assert(this.loaded, "Channel should be loaded when updating used routes");
Expand Down Expand Up @@ -582,7 +601,13 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
// returned in packagePath().
Object.freeze(this.pkg);

// Update the used routes of the channel in case GC was run before we were realized.
/**
* Update the used routes of the channel. If GC has run before this data store was realized, we will have
* the used routes saved. So, this will ensure that all the child contexts have up-to-date used routes as
* per the last time GC was run.
* Also, this data store may have been realized during summarize. In that case, the child contexts need to
* have their used routes updated to determine if its needs to summarize again and to add it to the summary.
*/
this.updateChannelUsedRoutes();

// And notify the pending promise it is now available
Expand Down
12 changes: 10 additions & 2 deletions packages/runtime/container-runtime/src/dataStores.ts
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,14 @@ export class DataStores implements IDisposable {
return builder.getSummaryTree();
}

/**
* Generates data used for garbage collection. It does the following:
* 1. Calls into each child data store context to get its GC data.
* 2. Prefixs the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
* idenfied as belonging to the child.
* 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
* the GC data of this channel.
*/
public async getGCData(): Promise<IGarbageCollectionData> {
const builder = new GCDataBuilder();
// Iterate over each store and get their GC data.
Expand All @@ -449,8 +457,8 @@ export class DataStores implements IDisposable {
return context.attachState === AttachState.Attached;
}).map(async ([contextId, context]) => {
const contextGCData = await context.getGCData();
// Prefix the child's id to the ids of GC nodes returned by it. This gradually builds the id of
// each node to be a path from the root.
// Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
// This also gradually builds the id of each node to be a path from the root.
builder.prefixAndAddNodes(contextId, contextGCData.gcNodes);
}));

Expand Down
10 changes: 10 additions & 0 deletions packages/runtime/datastore/src/channelContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,18 @@ export interface IChannelContext {

reSubmit(content: any, localOpMetadata: unknown): void;

/**
* Returns the data used for garbage collection. This includes a list of GC nodes that represent this context
* including any of its children. Each node has a set of outbound routes to other GC nodes in the document.
*/
getGCData(): Promise<IGarbageCollectionData>;

/**
* After GC has run, called to notify this context of routes that are used in it. These are used for the following:
* 1. To identify if this context is being referenced in the document or not.
* 2. To identify if this context or any of its children's used routes changed since last summary.
* 3. They are added to the summary generated by this context.
*/
updateUsedRoutes(usedRoutes: string[]): void;
}

Expand Down
16 changes: 13 additions & 3 deletions packages/runtime/datastore/src/dataStoreRuntime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,16 @@ IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext {
builder.addNode("/", this.getOutboundRoutes());
}

/**
* Generates data used for garbage collection. This includes a list of GC nodes that represent this channel
* including any of its child channel contexts. Each node has a set of outbound routes to other GC nodes in the
* document. It does the following:
* 1. Calls into each child context to get its GC data.
* 2. Prefixs the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
* idenfied as belonging to the child.
* 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
* the GC data of this channel.
*/
public async getGCData(): Promise<IGarbageCollectionData> {
const builder = new GCDataBuilder();
// Iterate over each channel context and get their GC data.
Expand All @@ -599,8 +609,8 @@ IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext {
return this.isChannelAttached(contextId);
}).map(async ([contextId, context]) => {
const contextGCData = await context.getGCData();
// Prefix the child's id to the ids of its GC nodes. This gradually builds the id of each node to be
// a path from the root.
// Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
// This also gradually builds the id of each node to be a path from the root.
builder.prefixAndAddNodes(contextId, contextGCData.gcNodes);
}));

Expand All @@ -609,7 +619,7 @@ IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext {
}

/**
* After GC has run, called to notify this channel of routes that are used in it. It call the child contexts to
* After GC has run, called to notify this channel of routes that are used in it. It calls the child contexts to
* update their used routes.
* @param usedRoutes - The routes that are used in all contexts in this channel.
*/
Expand Down
5 changes: 5 additions & 0 deletions packages/runtime/datastore/src/localChannelContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,11 @@ export class LocalChannelContext implements IChannelContext {
}
}

/**
* Returns the data used for garbage collection. This includes a list of GC nodes that represent this context.
* Each node has a set of outbound routes to other GC nodes in the document. This should be called only after
* the context has loaded.
*/
public async getGCData(): Promise<IGarbageCollectionData> {
assert(this.isLoaded && this.channel !== undefined, "Channel should be loaded to run GC");
return this.channel.getGCData();
Expand Down
27 changes: 22 additions & 5 deletions packages/runtime/datastore/src/remoteChannelContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,32 @@ export class RemoteChannelContext implements IChannelContext {
return this.channel;
}

/**
* Returns the data used for garbage collection. This includes a list of GC nodes that represent this context.
* Each node has a set of outbound routes to other GC nodes in the document.
* If there is no new data in this context since the last summary, previous GC data is used.
* If there is new data, the GC data is generated again (by calling getGCDataInternal).
*/
public async getGCData(): Promise<IGarbageCollectionData> {
return this.summarizerNode.getGCData();
}

/**
* Generates the data used for garbage collection. This is called when there is new data since last summary. It
* loads the context and calls into the channel to get its GC data.
*/
private async getGCDataInternal(): Promise<IGarbageCollectionData> {
const channel = await this.getChannel();
return channel.getGCData();
}

/**
* After GC has run, called to notify the context of routes used in it. These are used for the following:
* 1. To identify if this context is being referenced in the document or not.
* 2. To determine if it needs to re-summarize in case used routes changed since last summary.
* 3. These are added to the summary generated by the context.
* @param usedRoutes - The routes that are used in this context.
*/
public updateUsedRoutes(usedRoutes: string[]) {
/**
* Currently, DDSs are always considered referenced and are not garbage collected. Update the summarizer node's
Expand All @@ -250,9 +272,4 @@ export class RemoteChannelContext implements IChannelContext {
this.summarizerNode.updateUsedRoutes([""]);
}
}

private async getGCDataInternal(): Promise<IGarbageCollectionData> {
const channel = await this.getChannel();
return channel.getGCData();
}
}
4 changes: 2 additions & 2 deletions packages/runtime/runtime-definitions/src/dataStoreContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,8 @@ export interface IFluidDataStoreChannel extends
summarize(fullTree?: boolean, trackState?: boolean): Promise<IChannelSummarizeResult>;

/**
* Returns the GC data for this data store. It contains a list of GC nodes that contains references to
* other GC nodes.
* Returns the data used for garbage collection. This includes a list of GC nodes that represent this context
* including any of its children. Each node has a list of outbound routes to other GC nodes in the document.
*/
getGCData(): Promise<IGarbageCollectionData>;

Expand Down
18 changes: 17 additions & 1 deletion packages/runtime/runtime-definitions/src/summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,15 +133,18 @@ export interface ISummarizerNode {

/**
* Extends the functionality of ISummarizerNode to support garbage collection. It adds / udpates the following APIs:
* - usedRoutes - The routes in this node that are currently in use.
* - getGCData - A new API that can be used to get the garbage collection data for this node.
* - summarize - Added a trackState flag which indicates whether the summarizer node should track the state of the
* summary or not.
* - createChild - Added the following params:
* - getGCDataFn - This gets the GC data from the caller. This must be provided in order for getGCData to work.
* - getInitialGCDetailsFn - This gets the initial GC details from the caller.
* - isReferenced - This tells whether this node is referenced in the document or not.
* - updateUsedRoutes - Used to notify this node of routes that are currently in use in it.
*/
export interface ISummarizerNodeWithGC extends ISummarizerNode {
// This tells whether this node is in use or not. Unused node can be garbage collected and reclaimed.
/** The routes in this node that are currently in use. */
readonly usedRoutes: string[];

summarize(fullTree: boolean, trackState?: boolean): Promise<IContextSummarizeResult>;
Expand All @@ -164,8 +167,21 @@ export interface ISummarizerNodeWithGC extends ISummarizerNode {
): ISummarizerNodeWithGC;

getChild(id: string): ISummarizerNodeWithGC | undefined;

/**
* Returns this node's data that is used for garbage collection. This includes a list of GC nodes that represent
* this node. Each node has a set of outbound routes to other GC nodes in the document.
*/
getGCData(): Promise<IGarbageCollectionData>;

/** Tells whether this node is being referenced in this document or not. Unreferenced node will get GC'd */
isReferenced(): boolean;

/**
* After GC has run, called to notify this node of routes that are used in it. These are used for the following:
* 1. To identify if this node is being referenced in the document or not.
* 2. To identify if this node or any of its children's used routes changed since last summary.
*/
updateUsedRoutes(usedRoutes: string[]): void;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,10 @@ class SummaryNodeWithGC extends SummaryNode {
/**
* Extends the functionality of SummarizerNode to manage this node's garbage collection data:
* - Adds a new API `getGCData` to return GC data of this node.
* - Caches the result of getGCData method to be used if nothing changes between summaries.
* - Caches the result of `getGCData` to be used if nothing changes between summaries.
* - Adds GC data to the result of summarize.
* - Manages the used routes of this node. These are used to identify if this node is referenced in the document
* and to determine if the node's used state changed since last summary.
* - Adds trackState param to summarize. If trackState is false, it bypasses the SummarizerNode and calls
* directly into summarizeInternal method.
*/
Expand Down Expand Up @@ -141,8 +143,8 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
}

/**
* Returns the GC data of this node. If nothing has changed since the last time we summarized, it tries to reuse
* existing data.
* Returns the GC data of this node. If nothing has changed since last summary, it tries to reuse the data from
* the previous summary. Else, it gets new GC data from the underlying Fluid object.
*/
public async getGCData(): Promise<IGarbageCollectionData> {
assert(this.getGCDataFn !== undefined, "GC data cannot be retrieved without getGCDataFn");
Expand Down

0 comments on commit 7ef8fce

Please sign in to comment.