From 59f1e16abd5d8080396b5174c1bff19fec2c99fe Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 12:29:46 -0700 Subject: [PATCH 01/27] final changes for rehydration --- packages/dds/cell/src/cell.ts | 2 +- packages/dds/cell/src/cellFactory.ts | 12 ++++ packages/dds/counter/src/counter.ts | 2 +- packages/dds/counter/src/counterFactory.ts | 12 ++++ packages/dds/ink/src/ink.ts | 2 +- packages/dds/ink/src/inkFactory.ts | 15 +++++ packages/dds/map/src/directory.ts | 17 +++++- packages/dds/map/src/map.ts | 17 +++++- packages/dds/matrix/src/matrix.ts | 2 +- packages/dds/matrix/src/runtime.ts | 12 ++++ .../src/consensusOrderedCollection.ts | 2 +- .../src/consensusOrderedCollectionFactory.ts | 12 ++++ .../src/consensusRegisterCollection.ts | 2 +- .../src/consensusRegisterCollectionFactory.ts | 12 ++++ packages/dds/sequence/src/sequence.ts | 2 +- packages/dds/sequence/src/sequenceFactory.ts | 34 +++++++++++ .../sequence/src/sharedIntervalCollection.ts | 14 ++++- packages/dds/sequence/src/sparsematrix.ts | 12 ++++ .../shared-object-base/src/sharedObject.ts | 14 ++++- .../src/sharedSummaryBlock.ts | 2 +- .../src/sharedSummaryBlockFactory.ts | 16 ++++++ .../baseContainerRuntimeFactory.ts | 3 +- .../container-definitions/src/loader.ts | 13 ++++- .../loader/container-loader/src/container.ts | 56 ++++++++++++++----- .../loader/container-loader/src/loader.ts | 6 +- .../loader/driver-utils/src/readAndParse.ts | 15 +++++ .../src/webWorkerLoader.ts | 4 +- .../runtime/agent-scheduler/src/scheduler.ts | 3 +- .../src/channel.ts | 17 ++++++ .../src/componentRuntime.ts | 3 + .../component-runtime/src/channelContext.ts | 4 +- .../container-runtime/src/componentContext.ts | 8 ++- .../src/test/componentContext.spec.ts | 9 ++- .../src/test/componentCreation.spec.ts | 27 ++++++--- .../runtime/test-runtime-utils/src/mocks.ts | 4 ++ .../test/attachRegisterLocalApiTests.spec.ts | 13 ++++- .../src/test/detachedContainerTests.spec.ts | 44 +++++++++------ .../service-load-test/src/nodeStressTest.ts | 13 ++++- .../test/test-utils/src/testFluidComponent.ts | 3 +- .../webpack-component-loader/src/loader.ts | 7 ++- 40 files changed, 393 insertions(+), 74 deletions(-) diff --git a/packages/dds/cell/src/cell.ts b/packages/dds/cell/src/cell.ts index 7b46ef639719..2995891d3d5b 100644 --- a/packages/dds/cell/src/cell.ts +++ b/packages/dds/cell/src/cell.ts @@ -202,7 +202,7 @@ export class SharedCell extends SharedObject { const rawContent = await storage.read(snapshotFileName); diff --git a/packages/dds/cell/src/cellFactory.ts b/packages/dds/cell/src/cellFactory.ts index 6d96a0409299..0a6339ad8cab 100644 --- a/packages/dds/cell/src/cellFactory.ts +++ b/packages/dds/cell/src/cellFactory.ts @@ -8,6 +8,7 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { SharedCell } from "./cell"; import { ISharedCell } from "./interfaces"; @@ -44,6 +45,17 @@ export class CellFactory implements IChannelFactory { return cell; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const cell = new SharedCell(id, runtime, attributes); + await cell.loadLocal(objectStorage); + return cell; + } + public create(document: IFluidDataStoreRuntime, id: string): ISharedCell { const cell = new SharedCell(id, document, this.attributes); cell.initializeLocal(); diff --git a/packages/dds/counter/src/counter.ts b/packages/dds/counter/src/counter.ts index 0c44c943b317..d38a14e4105c 100644 --- a/packages/dds/counter/src/counter.ts +++ b/packages/dds/counter/src/counter.ts @@ -135,7 +135,7 @@ export class SharedCounter extends SharedObject implements * @returns - promise that resolved when the load is completed */ protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService): Promise { const rawContent = await storage.read(snapshotFileName); diff --git a/packages/dds/counter/src/counterFactory.ts b/packages/dds/counter/src/counterFactory.ts index fc40da7904b6..1d087dcc6456 100644 --- a/packages/dds/counter/src/counterFactory.ts +++ b/packages/dds/counter/src/counterFactory.ts @@ -8,6 +8,7 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { SharedCounter } from "./counter"; import { ISharedCounter } from "./interfaces"; @@ -44,6 +45,17 @@ export class CounterFactory implements IChannelFactory { return counter; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const counter = new SharedCounter(id, runtime, attributes); + await counter.loadLocal(objectStorage); + return counter; + } + public create(document: IFluidDataStoreRuntime, id: string): ISharedCounter { const counter = new SharedCounter(id, document, this.attributes); counter.initializeLocal(); diff --git a/packages/dds/ink/src/ink.ts b/packages/dds/ink/src/ink.ts index b279cd9658ed..4e33843dfde9 100644 --- a/packages/dds/ink/src/ink.ts +++ b/packages/dds/ink/src/ink.ts @@ -154,7 +154,7 @@ export class Ink extends SharedObject implements IInk { * {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore} */ protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService, ): Promise { const header = await storage.read(snapshotFileName); diff --git a/packages/dds/ink/src/inkFactory.ts b/packages/dds/ink/src/inkFactory.ts index b55bfc3c8e3c..5ccb5e06223d 100644 --- a/packages/dds/ink/src/inkFactory.ts +++ b/packages/dds/ink/src/inkFactory.ts @@ -8,6 +8,7 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { ISharedObject } from "@fluidframework/shared-object-base"; import { Ink } from "./ink"; @@ -61,6 +62,20 @@ export class InkFactory implements IChannelFactory { return ink; } + /** + * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} + */ + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const ink = new Ink(runtime, id, attributes); + await ink.loadLocal(objectStorage); + return ink; + } + /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/dds/map/src/directory.ts b/packages/dds/map/src/directory.ts index 414690f589d8..f0cff39e81ba 100644 --- a/packages/dds/map/src/directory.ts +++ b/packages/dds/map/src/directory.ts @@ -343,6 +343,21 @@ export class DirectoryFactory { return directory; } + /** + * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} + */ + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const directory = new SharedDirectory(id, runtime, attributes); + await directory.loadLocal(objectStorage); + + return directory; + } + /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ @@ -653,7 +668,7 @@ export class SharedDirectory extends SharedObject implem * {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore} */ protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService) { const header = await storage.read(snapshotFileName); const data = JSON.parse(fromBase64ToUtf8(header)); diff --git a/packages/dds/map/src/map.ts b/packages/dds/map/src/map.ts index 52f399dbaca5..24788106f43b 100644 --- a/packages/dds/map/src/map.ts +++ b/packages/dds/map/src/map.ts @@ -86,6 +86,21 @@ export class MapFactory implements IChannelFactory { return map; } + /** + * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} + */ + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const map = new SharedMap(id, runtime, attributes); + await map.loadLocal(objectStorage); + + return map; + } + /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ @@ -321,7 +336,7 @@ export class SharedMap extends SharedObject implements IShared * {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore} */ protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService) { const header = await storage.read(snapshotFileName); diff --git a/packages/dds/matrix/src/matrix.ts b/packages/dds/matrix/src/matrix.ts index 2b2e8515895a..782c55737606 100644 --- a/packages/dds/matrix/src/matrix.ts +++ b/packages/dds/matrix/src/matrix.ts @@ -462,7 +462,7 @@ export class SharedMatrix debug(`${this.id} is now disconnected`); } - protected async loadCore(branchId: string, storage: IChannelStorageService) { + protected async loadCore(branchId: string | undefined, storage: IChannelStorageService) { try { await this.rows.load(this.runtime, new ObjectStoragePartition(storage, SnapshotPath.rows), branchId); await this.cols.load(this.runtime, new ObjectStoragePartition(storage, SnapshotPath.cols), branchId); diff --git a/packages/dds/matrix/src/runtime.ts b/packages/dds/matrix/src/runtime.ts index 67468e726177..dcad915e66ae 100644 --- a/packages/dds/matrix/src/runtime.ts +++ b/packages/dds/matrix/src/runtime.ts @@ -9,6 +9,7 @@ import { IChannelServices, IChannel, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { pkgVersion } from "./packageVersion"; import { SharedMatrix } from "./matrix"; @@ -42,6 +43,17 @@ export class SharedMatrixFactory implements IChannelFactory { return matrix; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const matrix = new SharedMatrix(runtime, id, attributes); + await matrix.loadLocal(objectStorage); + return matrix; + } + public create(document: IFluidDataStoreRuntime, id: string): IChannel { const matrix = new SharedMatrix(document, id, this.attributes); matrix.initializeLocal(); diff --git a/packages/dds/ordered-collection/src/consensusOrderedCollection.ts b/packages/dds/ordered-collection/src/consensusOrderedCollection.ts index 12cf28a4fced..b42c44e2463c 100644 --- a/packages/dds/ordered-collection/src/consensusOrderedCollection.ts +++ b/packages/dds/ordered-collection/src/consensusOrderedCollection.ts @@ -273,7 +273,7 @@ export class ConsensusOrderedCollection } protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService): Promise { assert(this.jobTracking.size === 0); const rawContentTracking = await storage.read(snapshotFileNameTracking); diff --git a/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts b/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts index 799a42e84d17..832b9c1ec741 100644 --- a/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts +++ b/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts @@ -7,6 +7,7 @@ import { IChannelAttributes, IFluidDataStoreRuntime, IChannelServices, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { ConsensusQueue } from "./consensusQueue"; import { IConsensusOrderedCollection, IConsensusOrderedCollectionFactory } from "./interfaces"; @@ -43,6 +44,17 @@ export class ConsensusQueueFactory implements IConsensusOrderedCollectionFactory return collection; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const collection = new ConsensusQueue(id, runtime, attributes); + await collection.loadLocal(objectStorage); + return collection; + } + public create(document: IFluidDataStoreRuntime, id: string): IConsensusOrderedCollection { const collection = new ConsensusQueue(id, document, this.attributes); collection.initializeLocal(); diff --git a/packages/dds/register-collection/src/consensusRegisterCollection.ts b/packages/dds/register-collection/src/consensusRegisterCollection.ts index c20df9725d7a..37ea96ca393b 100644 --- a/packages/dds/register-collection/src/consensusRegisterCollection.ts +++ b/packages/dds/register-collection/src/consensusRegisterCollection.ts @@ -210,7 +210,7 @@ export class ConsensusRegisterCollection } protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService, ): Promise { const header = await storage.read(snapshotFileName); diff --git a/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts b/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts index d57d81a9390e..e3e22954a86c 100644 --- a/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts +++ b/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts @@ -7,6 +7,7 @@ import { IChannelAttributes, IFluidDataStoreRuntime, IChannelServices, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { ConsensusRegisterCollection } from "./consensusRegisterCollection"; import { IConsensusRegisterCollection, IConsensusRegisterCollectionFactory } from "./interfaces"; @@ -43,6 +44,17 @@ export class ConsensusRegisterCollectionFactory implements IConsensusRegisterCol return collection; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const collection = new ConsensusRegisterCollection(id, runtime, attributes); + await collection.loadLocal(objectStorage); + return collection; + } + public create(document: IFluidDataStoreRuntime, id: string): IConsensusRegisterCollection { const collection = new ConsensusRegisterCollection(id, document, ConsensusRegisterCollectionFactory.Attributes); collection.initializeLocal(); diff --git a/packages/dds/sequence/src/sequence.ts b/packages/dds/sequence/src/sequence.ts index a2889ba7e9a6..b6ce9577d481 100644 --- a/packages/dds/sequence/src/sequence.ts +++ b/packages/dds/sequence/src/sequence.ts @@ -487,7 +487,7 @@ export abstract class SharedSegmentSequence } protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService) { const header = await storage.read(snapshotFileName); diff --git a/packages/dds/sequence/src/sequenceFactory.ts b/packages/dds/sequence/src/sequenceFactory.ts index cf18ba2f772d..b114bf220df7 100644 --- a/packages/dds/sequence/src/sequenceFactory.ts +++ b/packages/dds/sequence/src/sequenceFactory.ts @@ -9,6 +9,7 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { ISharedObject } from "@fluidframework/shared-object-base"; import { pkgVersion } from "./packageVersion"; @@ -55,6 +56,17 @@ export class SharedStringFactory implements IChannelFactory { return sharedString; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const sharedString = new SharedString(runtime, id, attributes); + await sharedString.loadLocal(objectStorage); + return sharedString; + } + public create(document: IFluidDataStoreRuntime, id: string): SharedString { const sharedString = new SharedString(document, id, this.attributes); sharedString.initializeLocal(); @@ -101,6 +113,17 @@ export class SharedObjectSequenceFactory implements IChannelFactory { return sharedSeq; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const sharedSeq = new SharedObjectSequence(runtime, id, attributes); + await sharedSeq.loadLocal(objectStorage); + return sharedSeq; + } + public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedString = new SharedObjectSequence(document, id, this.attributes); sharedString.initializeLocal(); @@ -147,6 +170,17 @@ export class SharedNumberSequenceFactory implements IChannelFactory { return sharedSeq; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const sharedSeq = new SharedNumberSequence(runtime, id, attributes); + await sharedSeq.loadLocal(objectStorage); + return sharedSeq; + } + public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedString = new SharedNumberSequence(document, id, this.attributes); sharedString.initializeLocal(); diff --git a/packages/dds/sequence/src/sharedIntervalCollection.ts b/packages/dds/sequence/src/sharedIntervalCollection.ts index e91ef9277658..0fdb95039196 100644 --- a/packages/dds/sequence/src/sharedIntervalCollection.ts +++ b/packages/dds/sequence/src/sharedIntervalCollection.ts @@ -63,6 +63,18 @@ export class SharedIntervalCollectionFactory implements IChannelFactory { return map; } + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const map = new SharedIntervalCollection(id, runtime, attributes); + await map.loadLocal(objectStorage); + + return map; + } + public create(runtime: IFluidDataStoreRuntime, id: string): SharedIntervalCollection { const map = new SharedIntervalCollection( id, @@ -174,7 +186,7 @@ export class SharedIntervalCollection { + const sharedObject = new SparseMatrix(runtime, id, attributes); + await sharedObject.loadLocal(objectStorage); + return sharedObject; + } + public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedObject = new SparseMatrix(document, id, this.attributes); sharedObject.initializeLocal(); diff --git a/packages/dds/shared-object-base/src/sharedObject.ts b/packages/dds/shared-object-base/src/sharedObject.ts index 7250d1ad5b2f..9fa76cc2ca94 100644 --- a/packages/dds/shared-object-base/src/sharedObject.ts +++ b/packages/dds/shared-object-base/src/sharedObject.ts @@ -140,6 +140,18 @@ export abstract class SharedObject { + await this.loadCore( + undefined, + objectStorage); + } + /** * Initializes the object as a local, non-shared object. This object can become shared after * it is attached to the document. @@ -200,7 +212,7 @@ export abstract class SharedObject; /** diff --git a/packages/dds/shared-summary-block/src/sharedSummaryBlock.ts b/packages/dds/shared-summary-block/src/sharedSummaryBlock.ts index e535da2d586a..0c4ade6aa616 100644 --- a/packages/dds/shared-summary-block/src/sharedSummaryBlock.ts +++ b/packages/dds/shared-summary-block/src/sharedSummaryBlock.ts @@ -127,7 +127,7 @@ export class SharedSummaryBlock extends SharedObject implements ISharedSummaryBl * {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore} */ protected async loadCore( - branchId: string, + branchId: string | undefined, storage: IChannelStorageService): Promise { const rawContent = await storage.read(snapshotFileName); const contents = JSON.parse(fromBase64ToUtf8(rawContent)) as ISharedSummaryBlockDataSerializable; diff --git a/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts b/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts index 8bd7a196b5c2..bc4311ffe729 100644 --- a/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts +++ b/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts @@ -8,6 +8,7 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, + IChannelStorageService, } from "@fluidframework/component-runtime-definitions"; import { ISharedObject, @@ -63,6 +64,21 @@ export class SharedSummaryBlockFactory implements IChannelFactory { return sharedSummaryBlock; } + /** + * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} + */ + public async loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + objectStorage: IChannelStorageService, + attributes: IChannelAttributes, + ): Promise { + const sharedSummaryBlock = new SharedSummaryBlock(id, runtime, attributes); + await sharedSummaryBlock.loadLocal(objectStorage); + + return sharedSummaryBlock; + } + /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/framework/aqueduct/src/containerRuntimeFactories/baseContainerRuntimeFactory.ts b/packages/framework/aqueduct/src/containerRuntimeFactories/baseContainerRuntimeFactory.ts index 85ae2644cdce..dfe9722d97b2 100644 --- a/packages/framework/aqueduct/src/containerRuntimeFactories/baseContainerRuntimeFactory.ts +++ b/packages/framework/aqueduct/src/containerRuntimeFactories/baseContainerRuntimeFactory.ts @@ -77,7 +77,8 @@ export class BaseContainerRuntimeFactory implements // we register the runtime so developers of providers can use it in the factory pattern. dc.register(IContainerRuntime, runtime); - if (!runtime.existing) { + // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). + if (!(runtime.existing || context.baseSnapshot)) { // If it's the first time through. await this.containerInitializingFirstTime(runtime); } diff --git a/packages/loader/container-definitions/src/loader.ts b/packages/loader/container-definitions/src/loader.ts index b07630aa7afc..844a11a9f1d0 100644 --- a/packages/loader/container-definitions/src/loader.ts +++ b/packages/loader/container-definitions/src/loader.ts @@ -69,6 +69,17 @@ export interface ICodeAllowList { testSource(source: IResolvedFluidCodeDetails): Promise; } +/** + * Source to create the detached container from. Either needs the codeDetails or a snapshot to start from. + */ +export type IDetachedContainerSource = { + codeDetails: IFluidCodeDetails, + useSnapshot: false, +} | { + snapshot: string; + useSnapshot: true; +}; + export interface IContainerEvents extends IEvent { (event: "readonly", listener: (readonly: boolean) => void): void; (event: "connected", listener: (clientId: string) => void); @@ -148,7 +159,7 @@ export interface ILoader { * Creates a new container using the specified chaincode but in an unattached state. While unattached all * updates will only be local until the user explicitly attaches the container to a service provider. */ - createDetachedContainer(source: IFluidCodeDetails): Promise; + createDetachedContainer(source: IDetachedContainerSource): Promise; } export enum LoaderHeader { diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 3976637ae111..d9e43dba1cd3 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -29,6 +29,7 @@ import { ContainerWarning, IThrottlingWarning, AttachState, + IDetachedContainerSource, } from "@fluidframework/container-definitions"; import { performanceNow } from "@fluidframework/common-utils"; import { @@ -55,6 +56,7 @@ import { isOnline, ensureFluidResolvedUrl, combineAppAndProtocolSummary, + readAndParseFromBlobs, } from "@fluidframework/driver-utils"; import { CreateContainerError } from "@fluidframework/container-utils"; import { @@ -193,7 +195,7 @@ export class Container extends EventEmitterWithErrorHandling i options: any, scope: IFluidObject & IFluidObject, loader: Loader, - source: IFluidCodeDetails, + source: IDetachedContainerSource, serviceFactory: IDocumentServiceFactory, urlResolver: IUrlResolver, logger?: ITelemetryBaseLogger, @@ -207,8 +209,9 @@ export class Container extends EventEmitterWithErrorHandling i urlResolver, {}, logger); - await container.createDetached(source); - + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + source.useSnapshot ? await container.createDetachedFromSnapshot(JSON.parse(source.snapshot)) : + await container.createDetached(source.codeDetails); return container; } @@ -993,6 +996,23 @@ export class Container extends EventEmitterWithErrorHandling i this.propagateConnectionState(); } + private async createDetachedFromSnapshot(snapshotTree: ISnapshotTree) { + const attributes = await this.getDocumentAttributes(undefined, snapshotTree); + assert.strictEqual(attributes.sequenceNumber, 0, "Seq number in detached container should be 0!!"); + this.attachDeltaManagerOpHandler(attributes); + + // ...load in the existing quorum + // Initialize the protocol handler + this.protocolHandler = + await this.loadAndInitializeProtocolState(attributes, undefined, snapshotTree); + + await this.createDetachedContext(attributes, snapshotTree); + + this.loaded = true; + + this.propagateConnectionState(); + } + private async getDocumentStorageService(): Promise { if (!this.service) { throw new Error("Not attached"); @@ -1002,7 +1022,7 @@ export class Container extends EventEmitterWithErrorHandling i } private async getDocumentAttributes( - storage: IDocumentStorageService, + storage: IDocumentStorageService | undefined, tree: ISnapshotTree | undefined, ): Promise { if (!tree) { @@ -1019,7 +1039,8 @@ export class Container extends EventEmitterWithErrorHandling i ? tree.trees[".protocol"].blobs.attributes : tree.blobs[".attributes"]; - const attributes = await readAndParse(storage, attributesHash); + const attributes = storage ? await readAndParse(storage, attributesHash) + : readAndParseFromBlobs(tree.trees[".protocol"].blobs, attributesHash); // Back-compat for older summaries with no term if (attributes.term === undefined) { @@ -1031,7 +1052,7 @@ export class Container extends EventEmitterWithErrorHandling i private async loadAndInitializeProtocolState( attributes: IDocumentAttributes, - storage: IDocumentStorageService, + storage: IDocumentStorageService | undefined, snapshot: ISnapshotTree | undefined, ): Promise { let members: [string, ISequencedClient][] = []; @@ -1040,11 +1061,20 @@ export class Container extends EventEmitterWithErrorHandling i if (snapshot) { const baseTree = ".protocol" in snapshot.trees ? snapshot.trees[".protocol"] : snapshot; - [members, proposals, values] = await Promise.all([ - readAndParse<[string, ISequencedClient][]>(storage, baseTree.blobs.quorumMembers!), - readAndParse<[number, ISequencedProposal, string[]][]>(storage, baseTree.blobs.quorumProposals!), - readAndParse<[string, ICommittedProposal][]>(storage, baseTree.blobs.quorumValues!), - ]); + if (storage) { + [members, proposals, values] = await Promise.all([ + readAndParse<[string, ISequencedClient][]>(storage, baseTree.blobs.quorumMembers), + readAndParse<[number, ISequencedProposal, string[]][]>(storage, baseTree.blobs.quorumProposals), + readAndParse<[string, ICommittedProposal][]>(storage, baseTree.blobs.quorumValues), + ]); + } else { + members = readAndParseFromBlobs<[string, ISequencedClient][]>(snapshot.trees[".protocol"].blobs, + baseTree.blobs.quorumMembers); + proposals = readAndParseFromBlobs<[number, ISequencedProposal, string[]][]>( + snapshot.trees[".protocol"].blobs, baseTree.blobs.quorumProposals); + values = readAndParseFromBlobs<[string, ICommittedProposal][]>(snapshot.trees[".protocol"].blobs, + baseTree.blobs.quorumValues); + } } const protocolHandler = this.initializeProtocolState( @@ -1491,7 +1521,7 @@ export class Container extends EventEmitterWithErrorHandling i /** * Creates a new, unattached container context */ - private async createDetachedContext(attributes: IDocumentAttributes) { + private async createDetachedContext(attributes: IDocumentAttributes, snapshot?: ISnapshotTree) { this.pkg = this.getCodeDetailsFromQuorum(); if (!this.pkg) { throw new Error("pkg should be provided in create flow!!"); @@ -1507,7 +1537,7 @@ export class Container extends EventEmitterWithErrorHandling i this.scope, this.codeLoader, runtimeFactory, - { id: null, blobs: {}, commits: {}, trees: {} }, // TODO this will be from the offline store + snapshot ?? null, attributes, this.blobManager, new DeltaManagerProxy(this._deltaManager), diff --git a/packages/loader/container-loader/src/loader.ts b/packages/loader/container-loader/src/loader.ts index a858cb81525c..ea97540a6528 100644 --- a/packages/loader/container-loader/src/loader.ts +++ b/packages/loader/container-loader/src/loader.ts @@ -17,7 +17,7 @@ import { ILoader, IProxyLoaderFactory, LoaderHeader, - IFluidCodeDetails, + IDetachedContainerSource, } from "@fluidframework/container-definitions"; import { Deferred, performanceNow } from "@fluidframework/common-utils"; import { ChildLogger, DebugLogger, PerformanceEvent } from "@fluidframework/telemetry-utils"; @@ -99,7 +99,7 @@ export class RelativeLoader extends EventEmitter implements ILoader { return this.loader.request(request); } - public async createDetachedContainer(source: IFluidCodeDetails): Promise { + public async createDetachedContainer(source: IDetachedContainerSource): Promise { throw new Error("Relative loader should not create a detached container"); } @@ -168,7 +168,7 @@ export class Loader extends EventEmitter implements ILoader { } } - public async createDetachedContainer(source: IFluidCodeDetails): Promise { + public async createDetachedContainer(source: IDetachedContainerSource): Promise { debug(`Container creating in detached state: ${performanceNow()} `); return Container.create( diff --git a/packages/loader/driver-utils/src/readAndParse.ts b/packages/loader/driver-utils/src/readAndParse.ts index 33743b4ffc05..af926f4063f3 100644 --- a/packages/loader/driver-utils/src/readAndParse.ts +++ b/packages/loader/driver-utils/src/readAndParse.ts @@ -20,3 +20,18 @@ export async function readAndParse(storage: Pick(blobs: {[index: string]: string}, id: string): T { + const encoded = blobs[id]; + const decoded = Buffer + .from(encoded, "base64") + .toString(); + return JSON.parse(decoded) as T; +} diff --git a/packages/loader/execution-context-loader/src/webWorkerLoader.ts b/packages/loader/execution-context-loader/src/webWorkerLoader.ts index 210b183acaf5..c18a0e59d69f 100644 --- a/packages/loader/execution-context-loader/src/webWorkerLoader.ts +++ b/packages/loader/execution-context-loader/src/webWorkerLoader.ts @@ -9,7 +9,7 @@ import { IRequest, IResponse, } from "@fluidframework/component-core-interfaces"; -import { IContainer, ILoader, IFluidCodeDetails } from "@fluidframework/container-definitions"; +import { IContainer, ILoader, IDetachedContainerSource } from "@fluidframework/container-definitions"; import { IFluidResolvedUrl } from "@fluidframework/driver-definitions"; import Comlink from "comlink"; @@ -71,7 +71,7 @@ export class WebWorkerLoader implements ILoader, IFluidRunnable, IFluidRouter { return this.proxy.resolve(request); } - public async createDetachedContainer(source: IFluidCodeDetails): Promise { + public async createDetachedContainer(source: IDetachedContainerSource): Promise { return this.proxy.createDetachedContainer(source); } } diff --git a/packages/runtime/agent-scheduler/src/scheduler.ts b/packages/runtime/agent-scheduler/src/scheduler.ts index f58a321df2c3..ea71e910ccfd 100644 --- a/packages/runtime/agent-scheduler/src/scheduler.ts +++ b/packages/runtime/agent-scheduler/src/scheduler.ts @@ -36,7 +36,8 @@ class AgentScheduler extends EventEmitter implements IAgentScheduler, IFluidRout public static async load(runtime: IFluidDataStoreRuntime, context: IFluidDataStoreContext) { let root: ISharedMap; let scheduler: ConsensusRegisterCollection; - if (!runtime.existing) { + // Don't create if existing on storage or loaded detached from snapshot(ex. draft mode). + if (!(runtime.existing || runtime.baseSnapshot)) { root = SharedMap.create(runtime, "root"); root.bindToContext(); scheduler = ConsensusRegisterCollection.create(runtime); diff --git a/packages/runtime/component-runtime-definitions/src/channel.ts b/packages/runtime/component-runtime-definitions/src/channel.ts index 60a1cac6c34e..65f9eac3cded 100644 --- a/packages/runtime/component-runtime-definitions/src/channel.ts +++ b/packages/runtime/component-runtime-definitions/src/channel.ts @@ -184,4 +184,21 @@ export interface IChannelFactory { * for consistency. */ create(runtime: IFluidDataStoreRuntime, id: string): IChannel; + + /** + * Loads the given channel. This is similar to load however it is used to load a local channel from a snapshot. + * This call is only ever invoked internally as the only thing that is ever directly loaded is the document itself. + * @param runtime - Component runtime containing state/info/helper methods about the component. + * @param id - ID of the channel. + * @param storage - Storage service to read objects at a given path. This is not connected to storage + * endpoint but have blobs to read from. + * @param channelAttributes - The attributes for the the channel to be loaded. + * @returns The loaded object + */ + loadLocal( + runtime: IFluidDataStoreRuntime, + id: string, + storage: IChannelStorageService, + channelAttributes: Readonly, + ): Promise; } diff --git a/packages/runtime/component-runtime-definitions/src/componentRuntime.ts b/packages/runtime/component-runtime-definitions/src/componentRuntime.ts index bc2b6c93993c..191b497a8490 100644 --- a/packages/runtime/component-runtime-definitions/src/componentRuntime.ts +++ b/packages/runtime/component-runtime-definitions/src/componentRuntime.ts @@ -22,6 +22,7 @@ import { IDocumentMessage, IQuorum, ISequencedDocumentMessage, + ISnapshotTree, } from "@fluidframework/protocol-definitions"; import { IInboundSignalMessage, IProvideFluidDataStoreRegistry } from "@fluidframework/runtime-definitions"; import { IChannel } from "."; @@ -51,6 +52,8 @@ export interface IFluidDataStoreRuntime extends readonly existing: boolean; + readonly baseSnapshot: ISnapshotTree | undefined; + readonly parentBranch: string | null; readonly connected: boolean; diff --git a/packages/runtime/component-runtime/src/channelContext.ts b/packages/runtime/component-runtime/src/channelContext.ts index 4580b013ed74..26d9d0641e79 100644 --- a/packages/runtime/component-runtime/src/channelContext.ts +++ b/packages/runtime/component-runtime/src/channelContext.ts @@ -32,8 +32,8 @@ export function createServiceEndpoints( submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: () => void, storageService: IDocumentStorageService, - tree?: Promise, - extraBlobs?: Promise>, + tree?: Promise | undefined, + extraBlobs?: Promise> | undefined, ) { const deltaConnection = new ChannelDeltaConnection( id, diff --git a/packages/runtime/container-runtime/src/componentContext.ts b/packages/runtime/container-runtime/src/componentContext.ts index d648de67bb85..efe29f3bc5f6 100644 --- a/packages/runtime/container-runtime/src/componentContext.ts +++ b/packages/runtime/container-runtime/src/componentContext.ts @@ -47,7 +47,7 @@ import { SummaryTracker } from "@fluidframework/runtime-utils"; import { ContainerRuntime } from "./containerRuntime"; // Snapshot Format Version to be used in component attributes. -const currentSnapshotFormatVersion = "0.1"; +export const currentSnapshotFormatVersion = "0.1"; /** * Added IFluidDataStoretAttributes similar to IChannelAttributes which will tell @@ -620,12 +620,14 @@ export class LocalFluidDataStoreContext extends FluidDataStoreContext { scope: IFluidObject & IFluidObject, summaryTracker: SummaryTracker, bindComponent: (componentRuntime: IFluidDataStoreChannel) => void, + private readonly snapshotTree: ISnapshotTree | undefined, /** * @deprecated 0.16 Issue #1635 Use the IFluidDataStoreFactory creation methods instead to specify initial state */ public readonly createProps?: any, ) { - super(runtime, id, false, storage, scope, summaryTracker, BindState.NotBound, bindComponent, pkg); + super(runtime, id, false, storage, scope, summaryTracker, + snapshotTree ? BindState.Bound : BindState.NotBound, bindComponent, pkg); this.attachListeners(); } @@ -667,7 +669,7 @@ export class LocalFluidDataStoreContext extends FluidDataStoreContext { return { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion pkg: this.pkg!, - snapshot: undefined, + snapshot: this.snapshotTree, }; } } diff --git a/packages/runtime/container-runtime/src/test/componentContext.spec.ts b/packages/runtime/container-runtime/src/test/componentContext.spec.ts index 6a0d5e3ce762..c330c241ed25 100644 --- a/packages/runtime/container-runtime/src/test/componentContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/componentContext.spec.ts @@ -60,7 +60,8 @@ describe("Component Context Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); // eslint-disable-next-line @typescript-eslint/no-floating-promises localComponentContext.realize(); @@ -92,7 +93,8 @@ describe("Component Context Tests", () => { storage, scope, new SummaryTracker("", 0, 0), - attachCb); + attachCb, + undefined); await localComponentContext.realize() .catch((error) => { @@ -121,7 +123,8 @@ describe("Component Context Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); // eslint-disable-next-line @typescript-eslint/no-floating-promises localComponentContext.realize(); diff --git a/packages/runtime/container-runtime/src/test/componentCreation.spec.ts b/packages/runtime/container-runtime/src/test/componentCreation.spec.ts index 7a7f2560995e..02abbd494a73 100644 --- a/packages/runtime/container-runtime/src/test/componentCreation.spec.ts +++ b/packages/runtime/container-runtime/src/test/componentCreation.spec.ts @@ -106,7 +106,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await context.realize(); @@ -127,7 +128,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await context.realize(); @@ -148,7 +150,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextA.realize(); @@ -169,7 +172,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextB.realize(); @@ -190,7 +194,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextB.realize(); @@ -208,7 +213,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextC.realize(); @@ -229,7 +235,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextFake.realize(); @@ -250,7 +257,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextFake.realize(); @@ -271,7 +279,8 @@ describe("Component Creation Tests", () => { storage, scope, summaryTracker, - attachCb); + attachCb, + undefined); try { await contextC.realize(); diff --git a/packages/runtime/test-runtime-utils/src/mocks.ts b/packages/runtime/test-runtime-utils/src/mocks.ts index e82ca2cd8341..033944a7874a 100644 --- a/packages/runtime/test-runtime-utils/src/mocks.ts +++ b/packages/runtime/test-runtime-utils/src/mocks.ts @@ -394,6 +394,10 @@ export class MockFluidDataStoreRuntime extends EventEmitter this._local = local; } + public get baseSnapshot() { + return undefined; + } + private _disposed = false; public get disposed() { return this._disposed; } diff --git a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts index a2f536881bd2..35f419d9478a 100644 --- a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts @@ -5,7 +5,12 @@ import assert from "assert"; import { IRequest } from "@fluidframework/component-core-interfaces"; -import { IFluidCodeDetails, IProxyLoaderFactory, AttachState } from "@fluidframework/container-definitions"; +import { + IFluidCodeDetails, + IProxyLoaderFactory, + AttachState, + IDetachedContainerSource, +} from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; @@ -27,6 +32,10 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { package: "detachedContainerTestPackage1", config: {}, }; + const source: IDetachedContainerSource = { + codeDetails, + useSnapshot: false, + }; const mapId1 = "mapId1"; const mapId2 = "mapId2"; @@ -38,7 +47,7 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { `${name} should be ${attached ? "Attached" : "Detached"}`; async function createDetachedContainerAndGetRootComponent() { - const container = await loader.createDetachedContainer(codeDetails); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const defaultComponent = response.value; diff --git a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts index 1d1f276d15b8..007a93a3e947 100644 --- a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -5,7 +5,12 @@ import assert from "assert"; import { IRequest } from "@fluidframework/component-core-interfaces"; -import { IFluidCodeDetails, IProxyLoaderFactory, AttachState } from "@fluidframework/container-definitions"; +import { + IFluidCodeDetails, + IProxyLoaderFactory, + AttachState, + IDetachedContainerSource, +} from "@fluidframework/container-definitions"; import { ConnectionState, Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; @@ -36,7 +41,10 @@ describe("Detached Container", () => { package: "detachedContainerTestPackage", config: {}, }; - + const source: IDetachedContainerSource = { + codeDetails: pkg, + useSnapshot: false, + }; const sharedStringId = "ss1Key"; const sharedMapId = "sm1Key"; const crcId = "crc1Key"; @@ -91,7 +99,7 @@ describe("Detached Container", () => { }); it("Create detached container", async () => { - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); assert.strictEqual(container.attachState, AttachState.Detached, "Container should be detached"); assert.strictEqual(container.closed, false, "Container should be open"); assert.strictEqual(container.deltaManager.inbound.length, 0, "Inbound queue should be empty"); @@ -106,7 +114,7 @@ describe("Detached Container", () => { }); it("Attach detached container", async () => { - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); await container.attach(request); assert.strictEqual(container.attachState, AttachState.Attached, "Container should be attached"); assert.strictEqual(container.closed, false, "Container should be open"); @@ -115,7 +123,7 @@ describe("Detached Container", () => { }); it("Components in detached container", async () => { - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); if (response.mimeType !== "fluid/object" && response.status !== 200) { @@ -135,7 +143,7 @@ describe("Detached Container", () => { }); it("Components in attached container", async () => { - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const component = response.value as ITestFluidComponent; @@ -158,7 +166,7 @@ describe("Detached Container", () => { }); it("Load attached container and check for components", async () => { - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const component = response.value as ITestFluidComponent; @@ -196,7 +204,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared string", async () => { const ops = { pos1: 0, seg: "b", type: 0 }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.equal(type, MessageType.Operation); @@ -231,7 +239,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared map", async () => { const ops = { key: "1", type: "set", value: { type: "Plain", value: "b" } }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -261,7 +269,7 @@ describe("Detached Container", () => { it("Fire channel attach ops during container attach", async () => { const testChannelId = "testChannel1"; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.id, @@ -290,7 +298,7 @@ describe("Detached Container", () => { it("Fire component attach ops during container attach", async () => { const testComponentType = "default"; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); @@ -321,7 +329,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for consensus register collection", async () => { const op = { key: "1", type: "write", serializedValue: JSON.stringify("b"), refSeq: 0 }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -356,7 +364,7 @@ describe("Detached Container", () => { value: { type: "Plain", value: "b" }, }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -385,7 +393,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared cell", async () => { const op = { type: "setCell", value: { type: "Plain", value: "b" } }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -413,7 +421,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared ink", async () => { const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -447,7 +455,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for consensus ordered collection", async () => { const op = { opName: "add", value: JSON.stringify("s") }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -478,7 +486,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for sparse matrix", async () => { const seg = { items: ["s"] }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -514,7 +522,7 @@ describe("Detached Container", () => { it.skip("Fire ops during container attach for shared matrix", async () => { const op = { pos1: 0, seg: 9, type: 0, target: "rows" }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(pkg); + const container = await loader.createDetachedContainer(source); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, diff --git a/packages/test/service-load-test/src/nodeStressTest.ts b/packages/test/service-load-test/src/nodeStressTest.ts index 385170070d6b..d22000d7fad7 100644 --- a/packages/test/service-load-test/src/nodeStressTest.ts +++ b/packages/test/service-load-test/src/nodeStressTest.ts @@ -6,7 +6,11 @@ import fs from "fs"; import child_process from "child_process"; import commander from "commander"; -import { IProxyLoaderFactory, IFluidCodeDetails } from "@fluidframework/container-definitions"; +import { + IProxyLoaderFactory, + IFluidCodeDetails, + IDetachedContainerSource, +} from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { OdspDocumentServiceFactory, OdspDriverUrlResolver } from "@fluidframework/odsp-driver"; import { LocalCodeLoader } from "@fluidframework/test-utils"; @@ -32,6 +36,11 @@ const codeDetails: IFluidCodeDetails = { config: {}, }; +const source: IDetachedContainerSource = { + codeDetails, + useSnapshot: false, +}; + const codeLoader = new LocalCodeLoader([[codeDetails, fluidExport]]); const urlResolver = new OdspDriverUrlResolver(); const odspTokenManager = new OdspTokenManager(odspTokensCache); @@ -81,7 +90,7 @@ function createLoader(config: IConfig) { async function initialize(config: IConfig) { const loader = createLoader(config); - const container = await loader.createDetachedContainer(codeDetails); + const container = await loader.createDetachedContainer(source); container.on("error", (error) => { console.log(error); process.exit(-1); diff --git a/packages/test/test-utils/src/testFluidComponent.ts b/packages/test/test-utils/src/testFluidComponent.ts index 8ff6013e5044..d496bdeb5278 100644 --- a/packages/test/test-utils/src/testFluidComponent.ts +++ b/packages/test/test-utils/src/testFluidComponent.ts @@ -90,7 +90,8 @@ export class TestFluidComponent implements ITestFluidComponent { } private async initialize() { - if (!this.runtime.existing) { + // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). + if (!(this.runtime.existing || this.runtime.baseSnapshot !== undefined)) { this.root = SharedMap.create(this.runtime, "root"); this.factoryEntriesMap.forEach((sharedObjectFactory: IChannelFactory, key: string) => { diff --git a/packages/tools/webpack-component-loader/src/loader.ts b/packages/tools/webpack-component-loader/src/loader.ts index 74a32ff68eb8..dc79d3d5d2be 100644 --- a/packages/tools/webpack-component-loader/src/loader.ts +++ b/packages/tools/webpack-component-loader/src/loader.ts @@ -12,6 +12,7 @@ import { IFluidCodeResolver, IResolvedFluidCodeDetails, isFluidPackage, + IDetachedContainerSource, } from "@fluidframework/container-definitions"; import { Container } from "@fluidframework/container-loader"; import { IDocumentServiceFactory } from "@fluidframework/driver-definitions"; @@ -168,8 +169,12 @@ export async function start( if (!codeDetails) { throw new Error("Code details must be defined for detached mode!!"); } + const source: IDetachedContainerSource = { + codeDetails, + useSnapshot: false, + }; const loader = await baseHost1.getLoader(); - container1 = await loader.createDetachedContainer(codeDetails); + container1 = await loader.createDetachedContainer(source); const attachDiv = document.createElement("div"); const attachButton = document.createElement("button"); From 4bffb77032070e914cb1bf71d64f372ca17c1978 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 20:02:36 -0700 Subject: [PATCH 02/27] final changes for rehydration --- packages/dds/merge-tree/package.json | 1 + packages/dds/merge-tree/src/snapshotLoader.ts | 8 +- .../component-runtime/src/componentRuntime.ts | 88 ++++++-- .../src/localChannelContext.ts | 145 ++++++++++--- .../runtime/component-runtime/src/utils.ts | 32 +++ .../container-runtime/src/containerRuntime.ts | 101 +++++++-- .../runtime/container-runtime/src/utils.ts | 40 ++++ .../test/deRehydrateContainerTests.spec.ts | 203 +++++++++++++++++- 8 files changed, 538 insertions(+), 80 deletions(-) create mode 100644 packages/runtime/component-runtime/src/utils.ts create mode 100644 packages/runtime/container-runtime/src/utils.ts diff --git a/packages/dds/merge-tree/package.json b/packages/dds/merge-tree/package.json index 4e393cb1eef8..b6bedee38df0 100644 --- a/packages/dds/merge-tree/package.json +++ b/packages/dds/merge-tree/package.json @@ -54,6 +54,7 @@ "@fluidframework/common-utils": "^0.21.0", "@fluidframework/component-core-interfaces": "^0.25.0", "@fluidframework/component-runtime-definitions": "^0.25.0", + "@fluidframework/container-definitions": "^0.25.0", "@fluidframework/protocol-definitions": "^0.1010.0", "@fluidframework/telemetry-utils": "^0.25.0" }, diff --git a/packages/dds/merge-tree/src/snapshotLoader.ts b/packages/dds/merge-tree/src/snapshotLoader.ts index 0e052d36dc56..24a88ba4be09 100644 --- a/packages/dds/merge-tree/src/snapshotLoader.ts +++ b/packages/dds/merge-tree/src/snapshotLoader.ts @@ -9,6 +9,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils"; import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions"; import { IFluidDataStoreRuntime, IChannelStorageService } from "@fluidframework/component-runtime-definitions"; import { ITelemetryLogger } from "@fluidframework/common-definitions"; +import { AttachState } from "@fluidframework/container-definitions"; import { Client } from "./client"; import { NonCollabClient, UniversalSequenceNumber } from "./constants"; import { ISegment, MergeTree } from "./mergeTree"; @@ -142,11 +143,10 @@ export class SnapshotLoader { // a snapshot in any case (summary or attach message) // once we get a client id this will be called with that // clientId in the connected event - // TODO: this won't support rehydrating a detached container - // we need to think more holistically about the dds state machine - // now that we differentiate attached vs local + // However if we load a detached container from snapshot, then we don't supply a default clientId + // because we don't want to start collaboration. this.client.startOrUpdateCollaboration( - this.runtime.clientId ?? "snapshot", + this.runtime.attachState === AttachState.Detached ? undefined : this.runtime.clientId ?? "snapshot", // tslint:disable-next-line:no-suspicious-comment // TODO: Make 'minSeq' non-optional once the new snapshot format becomes the default? // (See https://github.com/microsoft/FluidFramework/issues/84) diff --git a/packages/runtime/component-runtime/src/componentRuntime.ts b/packages/runtime/component-runtime/src/componentRuntime.ts index 4cedb68caa27..d5e8034e63de 100644 --- a/packages/runtime/component-runtime/src/componentRuntime.ts +++ b/packages/runtime/component-runtime/src/componentRuntime.ts @@ -30,7 +30,7 @@ import { ChildLogger, raiseConnectedEvent, } from "@fluidframework/telemetry-utils"; -import { buildSnapshotTree } from "@fluidframework/driver-utils"; +import { buildSnapshotTree, readAndParseFromBlobs } from "@fluidframework/driver-utils"; import { TreeTreeEntry } from "@fluidframework/protocol-base"; import { IClientDetails, @@ -38,6 +38,7 @@ import { IQuorum, ISequencedDocumentMessage, ITreeEntry, + ITree, } from "@fluidframework/protocol-definitions"; import { IAttachMessage, @@ -49,11 +50,17 @@ import { SchedulerType, } from "@fluidframework/runtime-definitions"; import { generateHandleContextPath } from "@fluidframework/runtime-utils"; -import { IChannel, IFluidDataStoreRuntime, IChannelFactory } from "@fluidframework/component-runtime-definitions"; +import { + IChannel, + IFluidDataStoreRuntime, + IChannelFactory, + IChannelAttributes, +} from "@fluidframework/component-runtime-definitions"; import { v4 as uuid } from "uuid"; import { IChannelContext, snapshotChannel } from "./channelContext"; import { LocalChannelContext } from "./localChannelContext"; import { RemoteChannelContext } from "./remoteChannelContext"; +import { convertSnapshotToITree } from "./utils"; export enum ComponentMessageType { // Creates a new channel @@ -170,6 +177,9 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto private readonly notBoundedChannelContextSet = new Set(); private boundhandles: Set | undefined; private _attachState: AttachState; + // This set stores the id of unloaded local channels. This is meant to be used only in detached container when + // loaded from a snapshot. + private readonly unLoadedLocalChannel = new Set(); private constructor( private readonly componentContext: IFluidDataStoreContext, @@ -194,21 +204,40 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // Must always receive the component type inside of the attributes if (tree?.trees !== undefined) { Object.keys(tree.trees).forEach((path) => { - const channelContext = new RemoteChannelContext( - this, - componentContext, - componentContext.storage, - (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), - path, - tree.trees[path], - this.sharedObjectRegistry, - undefined /* extraBlobs */, - componentContext.branch, - this.componentContext.summaryTracker.createOrGetChild( + let channelContext: IChannelContext; + // If already exists on storage, then create a remote channel. However, if it is loaded from a snpashot + // but not yet exists on storage, then create a Local Channel. + if (existing) { + channelContext = new RemoteChannelContext( + this, + componentContext, + componentContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), + path, + tree.trees[path], + this.sharedObjectRegistry, + undefined /* extraBlobs */, + componentContext.branch, + this.componentContext.summaryTracker.createOrGetChild( + path, + this.deltaManager.lastSequenceNumber, + )); + } else { + const channelAttributes = readAndParseFromBlobs( + tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); + channelContext = new LocalChannelContext( path, - this.deltaManager.lastSequenceNumber, - )); + this.sharedObjectRegistry, + channelAttributes.type, + this, + this.componentContext, + this.componentContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), + tree.trees[path]); + this.unLoadedLocalChannel.add(path); + } const deferred = new Deferred(); deferred.resolve(channelContext); @@ -218,7 +247,9 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } this.attachListener(); - this.bindState = existing ? BindState.Bound : BindState.NotBound; + + // If exists on storage or loaded from a snapshot, it should already be binded. + this.bindState = existing || tree !== undefined ? BindState.Bound : BindState.NotBound; this._attachState = existing ? AttachState.Attached : AttachState.Detached; // If it's existing we know it has been attached. @@ -227,6 +258,10 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } } + public get baseSnapshot() { + return this.componentContext.baseSnapshot; + } + public dispose(): void { if (this._disposed) { return; @@ -250,7 +285,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const value = await this.contextsDeferred.get(id)!.promise; const channel = await value.getChannel(); - + this.unLoadedLocalChannel.delete(channel.id); return { mimeType: "fluid/object", status: 200, value: channel }; } @@ -279,7 +314,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const context = await this.contextsDeferred.get(id)!.promise; const channel = await context.getChannel(); - + this.unLoadedLocalChannel.delete(channel.id); return channel; } @@ -296,7 +331,8 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.componentContext, this.componentContext.storage, (content, localOpMetadata) => this.submitChannelOp(id, content, localOpMetadata), - (address: string) => this.setChannelDirty(address)); + (address: string) => this.setChannelDirty(address), + undefined); this.contexts.set(id, context); if (this.contextsDeferred.has(id)) { @@ -308,6 +344,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.contextsDeferred.set(id, deferred); } + assert(context.channel, "Channel should be loaded when created!!"); return context.channel; } @@ -528,7 +565,16 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } if (!this.notBoundedChannelContextSet.has(objectId)) { - const snapshot = value.getAttachSnapshot(); + let snapshot: ITree; + if (this.unLoadedLocalChannel.has(objectId)) { + // If this channel is not yet loaded, then there should be no changes in the snapshot from which + // it was created as it is detached container. So just use the previous snapshot. + assert(this.componentContext.baseSnapshot, + "BaseSnapshot should be there as detached container loaded from snapshot"); + snapshot = convertSnapshotToITree(this.componentContext.baseSnapshot.trees[objectId]); + } else { + snapshot = value.getAttachSnapshot(); + } // And then store the tree entries.push(new TreeTreeEntry(objectId, snapshot)); diff --git a/packages/runtime/component-runtime/src/localChannelContext.ts b/packages/runtime/component-runtime/src/localChannelContext.ts index fee40ddd39a5..ff5dfadbf88e 100644 --- a/packages/runtime/component-runtime/src/localChannelContext.ts +++ b/packages/runtime/component-runtime/src/localChannelContext.ts @@ -8,43 +8,63 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions"; import { ISequencedDocumentMessage, ITree, + ISnapshotTree, } from "@fluidframework/protocol-definitions"; -import { IChannel, IFluidDataStoreRuntime } from "@fluidframework/component-runtime-definitions"; +import { + IChannel, + IFluidDataStoreRuntime, + IChannelFactory, + IChannelAttributes, +} from "@fluidframework/component-runtime-definitions"; import { IFluidDataStoreContext } from "@fluidframework/runtime-definitions"; +import { readAndParse } from "@fluidframework/driver-utils"; +import { CreateContainerError } from "@fluidframework/container-utils"; import { createServiceEndpoints, IChannelContext, snapshotChannel } from "./channelContext"; import { ChannelDeltaConnection } from "./channelDeltaConnection"; import { ISharedObjectRegistry } from "./componentRuntime"; +import { ChannelStorageService } from "./channelStorageService"; /** * Channel context for a locally created channel */ export class LocalChannelContext implements IChannelContext { - public readonly channel: IChannel; + public channel: IChannel | undefined; + private isLoaded = false; private attached = false; - private connection: ChannelDeltaConnection | undefined; + private readonly pending: ISequencedDocumentMessage[] = []; + private _services: { + readonly deltaConnection: ChannelDeltaConnection, + readonly objectStorage: ChannelStorageService, + } | undefined; private readonly dirtyFn: () => void; + private readonly factory: IChannelFactory | undefined; constructor( - id: string, + private readonly id: string, registry: ISharedObjectRegistry, type: string, - runtime: IFluidDataStoreRuntime, + private readonly runtime: IFluidDataStoreRuntime, private readonly componentContext: IFluidDataStoreContext, private readonly storageService: IDocumentStorageService, private readonly submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: (address: string) => void, + private readonly snapshotTree: ISnapshotTree | undefined, ) { - const factory = registry.get(type); - if (factory === undefined) { + this.factory = registry.get(type); + if (this.factory === undefined) { throw new Error(`Channel Factory ${type} not registered`); } - - this.channel = factory.create(runtime, id); - + if (snapshotTree === undefined) { + this.channel = this.factory.create(runtime, id); + this.isLoaded = true; + } this.dirtyFn = () => { dirtyFn(id); }; } public async getChannel(): Promise { + if (this.channel === undefined) { + this.channel = await this.loadChannel(); + } return this.channel; } @@ -53,23 +73,23 @@ export class LocalChannelContext implements IChannelContext { if (!this.attached) { return; } - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.setConnectionState(connected); + this.services.deltaConnection.setConnectionState(connected); } public processOp(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void { assert(this.attached, "Local channel must be attached when processing op"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.process(message, local, localOpMetadata); + if (this.isLoaded) { + this.services.deltaConnection.process(message, local, localOpMetadata); + } else { + this.pending.push(message); + } } public reSubmit(content: any, localOpMetadata: unknown) { + assert(this.isLoaded, "Channel should be loaded to resubmit ops"); assert(this.attached, "Local channel must be attached when resubmitting op"); - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.reSubmit(content, localOpMetadata); + this.services.deltaConnection.reSubmit(content, localOpMetadata); } public async snapshot(fullTree: boolean = false): Promise { @@ -77,23 +97,94 @@ export class LocalChannelContext implements IChannelContext { } public getAttachSnapshot(): ITree { + assert(this.isLoaded && this.channel !== undefined, "Channel should be loaded to take snapshot"); return snapshotChannel(this.channel); } + private async loadChannel(): Promise { + assert(!this.isLoaded, "Channel must not already be loaded when loading"); + assert(this.snapshotTree, "Snapshot should be provided to load from!!"); + + assert(await this.services.objectStorage.contains(".attributes"), ".attributes blob should be present"); + const attributes = await readAndParse( + this.services.objectStorage, + ".attributes"); + + assert(this.factory, "Factory should be there for local channel"); + const channel = await this.factory.loadLocal( + this.runtime, + this.id, + this.services.objectStorage, + attributes); + + // Commit changes. + this.channel = channel; + this.isLoaded = true; + + if (this.attached) { + this.channel.connect(this.services); + } + + // Send all pending messages to the channel + for (const message of this.pending) { + try { + this.services.deltaConnection.process(message, false, undefined /* localOpMetadata */); + } catch (err) { + // record sequence number for easier debugging + const error = CreateContainerError(err); + error.sequenceNumber = message.sequenceNumber; + throw error; + } + } + return this.channel; + } + public attach(): void { if (this.attached) { throw new Error("Channel is already attached"); } - const services = createServiceEndpoints( - this.channel.id, - this.componentContext.connected, - this.submitFn, - this.dirtyFn, - this.storageService); - this.connection = services.deltaConnection; - this.channel.connect(services); - + if (this.isLoaded) { + assert(this.channel, "Channel should be there if loaded!!"); + this.channel.connect(this.services); + } this.attached = true; } + + private get services() { + if (this._services === undefined) { + let blobMap: Map | undefined; + if (this.snapshotTree !== undefined) { + blobMap = new Map(); + this.collectExtraBlobsAndSanitizeSnapshot(this.snapshotTree, blobMap); + } + this._services = createServiceEndpoints( + this.id, + this.componentContext.connected, + this.submitFn, + this.dirtyFn, + this.storageService, + this.snapshotTree !== undefined ? Promise.resolve(this.snapshotTree) : undefined, + blobMap !== undefined ? + Promise.resolve(blobMap) : undefined, + ); + } + return this._services; + } + + private collectExtraBlobsAndSanitizeSnapshot(snapshotTree: ISnapshotTree, blobMap: Map) { + const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); + for (const [blobName, blobId] of blobMapInitial.entries()) { + const blobValue = blobMapInitial.get(blobId); + if (blobValue !== undefined) { + blobMap.set(blobId, blobValue); + } else { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete snapshotTree.blobs[blobName]; + } + } + for (const value of Object.values(snapshotTree.trees)) { + this.collectExtraBlobsAndSanitizeSnapshot(value, blobMap); + } + } } diff --git a/packages/runtime/component-runtime/src/utils.ts b/packages/runtime/component-runtime/src/utils.ts new file mode 100644 index 000000000000..4ca462965d7c --- /dev/null +++ b/packages/runtime/component-runtime/src/utils.ts @@ -0,0 +1,32 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +import { Buffer } from "buffer"; +import { ISnapshotTree, ITreeEntry, ITree } from "@fluidframework/protocol-definitions"; +import { BlobTreeEntry, TreeTreeEntry } from "@fluidframework/protocol-base"; + +export function convertSnapshotToITree(snapshotTree: ISnapshotTree): ITree { + const entries: ITreeEntry[] = []; + const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); + const blobMapFinal = new Map(); + for (const [key, value] of blobMapInitial.entries()) { + if (blobMapInitial.has(value)) { + blobMapFinal[key] = blobMapInitial.get(value); + } + } + for (const [key, value] of Object.entries(blobMapFinal)) { + const decoded = Buffer.from(value, "base64").toString(); + entries.push(new BlobTreeEntry(key, decoded)); + } + for (const [key, tree] of Object.entries(snapshotTree.trees)) { + entries.push(new TreeTreeEntry(key, convertSnapshotToITree(tree))); + } + const finalTree: ITree = { + entries, + // eslint-disable-next-line no-null/no-null + id: null, + }; + return finalTree; +} diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index bd251dc974e5..2433aa0dc57e 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -44,6 +44,7 @@ import { BlobCacheStorageService, buildSnapshotTree, readAndParse, + readAndParseFromBlobs, } from "@fluidframework/driver-utils"; import { CreateContainerError } from "@fluidframework/container-utils"; import { @@ -78,9 +79,19 @@ import { NamedFluidDataStoreRegistryEntries, SchedulerType, } from "@fluidframework/runtime-definitions"; -import { FluidSerializer, SummaryTracker, RequestParser } from "@fluidframework/runtime-utils"; +import { + FluidSerializer, + SummaryTracker, + RequestParser, +} from "@fluidframework/runtime-utils"; import { v4 as uuid } from "uuid"; -import { FluidDataStoreContext, LocalFluidDataStoreContext, RemotedFluidDataStoreContext } from "./componentContext"; +import { + FluidDataStoreContext, + LocalFluidDataStoreContext, + RemotedFluidDataStoreContext, + currentSnapshotFormatVersion, + IFluidDataStoretAttributes, +} from "./componentContext"; import { FluidHandleContext } from "./componentHandleContext"; import { FluidDataStoreRegistry } from "./componentRegistry"; import { debug } from "./debug"; @@ -93,6 +104,7 @@ import { ReportOpPerfTelemetry } from "./connectionTelemetry"; import { SummaryCollection } from "./summaryCollection"; import { PendingStateManager } from "./pendingStateManager"; import { pkgVersion } from "./packageVersion"; +import { convertSnapshotToSummaryTree } from "./utils"; const chunksBlobName = ".chunks"; @@ -455,10 +467,9 @@ export class ContainerRuntime extends EventEmitter const componentRegistry = new ContainerRuntimeDataStoreRegistry(registryEntries); const chunkId = context.baseSnapshot?.blobs[chunksBlobName]; - const chunks = chunkId - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ? await readAndParse<[string, string[]][]>(context.storage!, chunkId) - : []; + const chunks = context.baseSnapshot && chunkId ? context.storage ? + await readAndParse<[string, string[]][]>(context.storage, chunkId) : + readAndParseFromBlobs<[string, string[]][]>(context.baseSnapshot.blobs, chunkId) : []; const runtime = new ContainerRuntime( context, @@ -468,8 +479,9 @@ export class ContainerRuntime extends EventEmitter containerScope, requestHandler); - // Create all internal components in first load. - if (!context.existing) { + // Create all internal components if not already existing on storage or loaded a detached + // container from snapshot(ex. draft mode). + if (!(context.existing || context.baseSnapshot !== null)) { await runtime.createRootDataStore(schedulerId, schedulerId); } @@ -575,6 +587,9 @@ export class ContainerRuntime extends EventEmitter private latestSummaryAck: ISummaryContext; private readonly summaryTracker: SummaryTracker; private readonly notBoundedComponentContexts = new Set(); + // This set stores the id of unrealized components. This is meant to be used only in detached container when loaded + // from a snapshot. + private readonly unRealizedComponents = new Set(); private tasks: string[] = []; @@ -667,13 +682,44 @@ export class ContainerRuntime extends EventEmitter // Create a context for each of them for (const [key, value] of components) { - const componentContext = new RemotedFluidDataStoreContext( - key, - typeof value === "string" ? value : Promise.resolve(value), - this, - this.storage, - this.containerScope, - this.summaryTracker.createOrGetChild(key, this.summaryTracker.referenceSequenceNumber)); + let componentContext: FluidDataStoreContext; + // If it is loaded from a snapshot but in detached state, then create a local component. + if (this.attachState === AttachState.Detached) { + let pkgFromSnapshot: string[]; + if (context.baseSnapshot === null) { + throw new Error("Snapshot should be there to load from!!"); + } + const snapshotTree = value as ISnapshotTree; + // Need to rip through snapshot. + const { pkg, snapshotFormatVersion } = readAndParseFromBlobs( + snapshotTree.blobs, + snapshotTree.blobs[".component"]); + // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. + // For snapshotFormatVersion = "0.1", pkg is jsonified, otherwise it is just a string. + if (snapshotFormatVersion === currentSnapshotFormatVersion) { + pkgFromSnapshot = JSON.parse(pkg) as string[]; + } else { + throw new Error(`Invalid snapshot format version ${snapshotFormatVersion}`); + } + this.unRealizedComponents.add(key); + componentContext = new LocalFluidDataStoreContext( + key, + pkgFromSnapshot, + this, + this.storage, + this.containerScope, + this.summaryTracker.createOrGetChild(key, this.deltaManager.lastSequenceNumber), + (cr: IFluidDataStoreChannel) => this.bindComponent(cr), + snapshotTree); + } else { + componentContext = new RemotedFluidDataStoreContext( + key, + typeof value === "string" ? value : Promise.resolve(value), + this, + this.storage, + this.containerScope, + this.summaryTracker.createOrGetChild(key, this.summaryTracker.referenceSequenceNumber)); + } this.setNewContext(key, componentContext); } @@ -1061,6 +1107,7 @@ export class ContainerRuntime extends EventEmitter } const componentContext = await deferredContext.promise; + this.unRealizedComponents.delete(id); return componentContext.realize(); } @@ -1165,6 +1212,7 @@ export class ContainerRuntime extends EventEmitter this.containerScope, this.summaryTracker.createOrGetChild(id, this.deltaManager.lastSequenceNumber), (cr: IFluidDataStoreChannel) => this.bindComponent(cr), + undefined, undefined); const deferred = new Deferred(); @@ -1191,6 +1239,7 @@ export class ContainerRuntime extends EventEmitter this.containerScope, this.summaryTracker.createOrGetChild(id, this.deltaManager.lastSequenceNumber), (cr: IFluidDataStoreChannel) => this.bindComponent(cr), + undefined, undefined /* #1635: Remove LocalFluidDataStoreContext createProps */); const deferred = new Deferred(); @@ -1459,13 +1508,21 @@ export class ContainerRuntime extends EventEmitter !(this.notBoundedComponentContexts.has(key) || summaryTree.tree[key]), ) .map(([key, value]) => { - const snapshot = value.generateAttachMessage().snapshot; - const treeWithStats = this.summaryTreeConverter.convertToSummaryTree( - snapshot, - `/${encodeURIComponent(key)}`, - true, - ); - summaryTree.tree[key] = treeWithStats.summaryTree; + if (this.unRealizedComponents.has(key)) { + // If this component is not yet loaded, then there should be no changes in the snapshot from + // which it was created as it is detached container. So just use the previous snapshot. + assert(this.context.baseSnapshot, + "BaseSnapshot should be there as detached container loaded from snapshot"); + summaryTree.tree[key] = convertSnapshotToSummaryTree(this.context.baseSnapshot.trees[key]); + } else { + const snapshot = value.generateAttachMessage().snapshot; + const treeWithStats = this.summaryTreeConverter.convertToSummaryTree( + snapshot, + `/${encodeURIComponent(key)}`, + true, + ); + summaryTree.tree[key] = treeWithStats.summaryTree; + } }); } while (notBoundedComponentContextsLength !== this.notBoundedComponentContexts.size); diff --git a/packages/runtime/container-runtime/src/utils.ts b/packages/runtime/container-runtime/src/utils.ts new file mode 100644 index 000000000000..a8458c7475da --- /dev/null +++ b/packages/runtime/container-runtime/src/utils.ts @@ -0,0 +1,40 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +import { Buffer } from "buffer"; +import { + SummaryObject, + ISummaryTree, + ISnapshotTree, + ISummaryBlob, + SummaryType, +} from "@fluidframework/protocol-definitions"; + +export function convertSnapshotToSummaryTree(snapshotTree: ISnapshotTree): ISummaryTree { + const entries: {[index: string]: SummaryObject} = {}; + const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); + const blobMapFinal = new Map(); + for (const [key, value] of blobMapInitial.entries()) { + if (blobMapInitial.has(value)) { + blobMapFinal[key] = blobMapInitial.get(value); + } + } + for (const [key, value] of Object.entries(blobMapFinal)) { + const decoded = Buffer.from(value, "base64").toString(); + const summaryBlob: ISummaryBlob = { + content: decoded, + type: SummaryType.Blob, + }; + entries[key] = summaryBlob; + } + for (const [key, tree] of Object.entries(snapshotTree.trees)) { + entries[key] = convertSnapshotToSummaryTree(tree); + } + const summaryTree: ISummaryTree = { + tree: entries, + type: SummaryType.Tree, + }; + return summaryTree; +} diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 9f531bde5552..c6e3665a31cd 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -4,7 +4,11 @@ */ import assert from "assert"; -import { IFluidCodeDetails, IProxyLoaderFactory } from "@fluidframework/container-definitions"; +import { + IFluidCodeDetails, + IProxyLoaderFactory, + IDetachedContainerSource, +} from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; @@ -15,23 +19,46 @@ import { ITestFluidComponent, TestFluidComponent, } from "@fluidframework/test-utils"; -import { SharedMap } from "@fluidframework/map"; +import { SharedMap, SharedDirectory } from "@fluidframework/map"; import { IDocumentAttributes } from "@fluidframework/protocol-definitions"; import { IContainerRuntime } from "@fluidframework/container-runtime-definitions"; import { IContainerRuntimeBase } from "@fluidframework/runtime-definitions"; +import { IRequest } from "@fluidframework/component-core-interfaces"; +import { SharedString, SparseMatrix } from "@fluidframework/sequence"; +import { ConsensusRegisterCollection } from "@fluidframework/register-collection"; +import { ConsensusQueue, ConsensusOrderedCollection } from "@fluidframework/ordered-collection"; +import { SharedMatrix } from "@fluidframework/matrix"; +import { SharedCell } from "@fluidframework/cell"; +import { Ink } from "@fluidframework/ink"; +import { SharedCounter } from "@fluidframework/counter"; describe(`Dehydrate Rehydrate Container Test`, () => { + const documentId = "deReHydrateContainerTest"; const codeDetails: IFluidCodeDetails = { package: "detachedContainerTestPackage1", config: {}, }; - const mapId1 = "mapId1"; + const source: IDetachedContainerSource = { + codeDetails, + useSnapshot: false, + }; + const sharedStringId = "ss1Key"; + const sharedMapId = "sm1Key"; + const crcId = "crc1Key"; + const cocId = "coc1Key"; + const sharedDirectoryId = "sd1Key"; + const sharedCellId = "scell1Key"; + const sharedMatrixId = "smatrix1Key"; + const sharedInkId = "sink1Key"; + const sparseMatrixId = "sparsematrixKey"; + const sharedCounterId = "sharedcounterKey"; let testDeltaConnectionServer: ILocalDeltaConnectionServer; let loader: Loader; + let request: IRequest; async function createDetachedContainerAndGetRootComponent() { - const container = await loader.createDetachedContainer(codeDetails); + const container = await loader.createDetachedContainer(source); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const defaultComponent = response.value; @@ -43,8 +70,18 @@ describe(`Dehydrate Rehydrate Container Test`, () => { function createTestLoader(urlResolver: IUrlResolver): Loader { const factory: TestFluidComponentFactory = new TestFluidComponentFactory([ - [mapId1, SharedMap.getFactory()], + [sharedStringId, SharedString.getFactory()], + [sharedMapId, SharedMap.getFactory()], + [crcId, ConsensusRegisterCollection.getFactory()], + [sharedDirectoryId, SharedDirectory.getFactory()], + [sharedCellId, SharedCell.getFactory()], + [sharedInkId, Ink.getFactory()], + [sharedMatrixId, SharedMatrix.getFactory()], + [cocId, ConsensusQueue.getFactory()], + [sparseMatrixId, SparseMatrix.getFactory()], + [sharedCounterId, SharedCounter.getFactory()], ]); + const codeLoader = new LocalCodeLoader([[codeDetails, factory]]); const documentServiceFactory = new LocalDocumentServiceFactory(testDeltaConnectionServer); return new Loader( @@ -72,6 +109,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { beforeEach(async () => { testDeltaConnectionServer = LocalDeltaConnectionServer.create(); const urlResolver = new LocalResolver(); + request = urlResolver.createCreateNewRequest(documentId); loader = createTestLoader(urlResolver); }); @@ -142,7 +180,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const component2 = peerComponent.peerComponent as TestFluidComponent; // Create a channel - const rootOfComponent1 = await (defaultComponent as TestFluidComponent).getSharedObject(mapId1); + const rootOfComponent1 = + await (defaultComponent as TestFluidComponent).getSharedObject(sharedMapId); rootOfComponent1.set("component2", component2.handle); const snapshotTree = JSON.parse(container.serialize()); @@ -150,4 +189,156 @@ describe(`Dehydrate Rehydrate Container Test`, () => { assert.strictEqual(Object.keys(snapshotTree.trees).length, 4, "4 trees should be there"); assert(snapshotTree.trees[component2.runtime.id], "Handle Bounded component should be in summary"); }); + + it("Rehydrate container from snapshot and check contents before attach", async () => { + const { container } = + await createDetachedContainerAndGetRootComponent(); + + const snapshotTree: string = container.serialize(); + const rehydrationSource: IDetachedContainerSource = { + snapshot: snapshotTree, + useSnapshot: true, + }; + + const container2 = await loader.createDetachedContainer(rehydrationSource); + + // Check for scheduler + const schedulerResponse = await container2.request({ url: "_scheduler" }); + assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); + const schedulerComponent = schedulerResponse.value as ITestFluidComponent; + assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + + // Check for default component + const response = await container2.request({ url: "/" }); + assert.strictEqual(response.status, 200, "Component should exist!!"); + const defaultComponent = response.value as ITestFluidComponent; + assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + + // Check for dds + const sharedMap = await defaultComponent.getSharedObject(sharedMapId); + const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); + const sharedString = await defaultComponent.getSharedObject(sharedStringId); + const sharedCell = await defaultComponent.getSharedObject(sharedCellId); + const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); + const crc = await defaultComponent.getSharedObject>(crcId); + const coc = await defaultComponent.getSharedObject(cocId); + const ink = await defaultComponent.getSharedObject(sharedInkId); + const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); + assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); + assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); + assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); + assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); + assert.strictEqual(crc.id, crcId, "CRC should exist!!"); + assert.strictEqual(coc.id, cocId, "COC should exist!!"); + assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); + assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); + assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); + }); + + it("Rehydrate container from snapshot and check contents after attach", async () => { + const { container } = + await createDetachedContainerAndGetRootComponent(); + + const snapshotTree: string = container.serialize(); + const rehydrationSource: IDetachedContainerSource = { + snapshot: snapshotTree, + useSnapshot: true, + }; + + const container2 = await loader.createDetachedContainer(rehydrationSource); + await container2.attach(request); + + // Check for scheduler + const schedulerResponse = await container2.request({ url: "_scheduler" }); + assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); + const schedulerComponent = schedulerResponse.value as ITestFluidComponent; + assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + + // Check for default component + const response = await container2.request({ url: "/" }); + assert.strictEqual(response.status, 200, "Component should exist!!"); + const defaultComponent = response.value as ITestFluidComponent; + assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + + // Check for dds + const sharedMap = await defaultComponent.getSharedObject(sharedMapId); + const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); + const sharedString = await defaultComponent.getSharedObject(sharedStringId); + const sharedCell = await defaultComponent.getSharedObject(sharedCellId); + const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); + const crc = await defaultComponent.getSharedObject>(crcId); + const coc = await defaultComponent.getSharedObject(cocId); + const ink = await defaultComponent.getSharedObject(sharedInkId); + const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); + assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); + assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); + assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); + assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); + assert.strictEqual(crc.id, crcId, "CRC should exist!!"); + assert.strictEqual(coc.id, cocId, "COC should exist!!"); + assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); + assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); + assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); + }); + + it("Change contents of dds, then rehydrate and then check snapshot", async () => { + const { container } = + await createDetachedContainerAndGetRootComponent(); + + const responseBefore = await container.request({ url: "/" }); + const defaultComponentBefore = responseBefore.value as ITestFluidComponent; + const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); + sharedStringBefore.insertText(0, "Hello"); + + const snapshotTree: string = container.serialize(); + const rehydrationSource: IDetachedContainerSource = { + snapshot: snapshotTree, + useSnapshot: true, + }; + const container2 = await loader.createDetachedContainer(rehydrationSource); + + const responseAfter = await container2.request({ url: "/" }); + const defaultComponentAfter = responseAfter.value as ITestFluidComponent; + const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); + assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), + "Snapshot of shared string should match and contents should be same!!"); + }); + + it("Rehydrate container from snapshot, change contents of dds and then check snapshot", async () => { + const { container } = + await createDetachedContainerAndGetRootComponent(); + let str = "AA"; + const response1 = await container.request({ url: "/" }); + const defaultComponent1 = response1.value as ITestFluidComponent; + const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); + sharedString1.insertText(0, str); + const snapshotTree: string = container.serialize(); + const rehydrationSource: IDetachedContainerSource = { + snapshot: snapshotTree, + useSnapshot: true, + }; + + const container2 = await loader.createDetachedContainer(rehydrationSource); + const responseBefore = await container2.request({ url: "/" }); + const defaultComponentBefore = responseBefore.value as ITestFluidComponent; + const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); + const sharedMapBefore = await defaultComponentBefore.getSharedObject(sharedMapId); + str += "BB"; + sharedStringBefore.insertText(0, str); + sharedMapBefore.set("0", str); + + await container2.attach(request); + const responseAfter = await container2.request({ url: "/" }); + const defaultComponentAfter = responseAfter.value as ITestFluidComponent; + const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); + const sharedMapAfter = await defaultComponentAfter.getSharedObject(sharedMapId); + assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), + "Snapshot of shared string should match and contents should be same!!"); + assert.strictEqual(JSON.stringify(sharedMapAfter.snapshot()), JSON.stringify(sharedMapBefore.snapshot()), + "Snapshot of shared map should match and contents should be same!!"); + }); }); From 59d66b5ba8520dfedcfc938ff34a88c2f6574b78 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 21:44:49 -0700 Subject: [PATCH 03/27] mertge conflict --- .../loader/container-loader/src/container.ts | 4 +- .../container-runtime/src/containerRuntime.ts | 14 +- .../runtime/datastore/src/dataStoreRuntime.ts | 93 ++++++++--- .../datastore/src/localChannelContext.ts | 149 ++++++++++++++---- packages/runtime/datastore/src/utils.ts | 32 ++++ 5 files changed, 236 insertions(+), 56 deletions(-) create mode 100644 packages/runtime/datastore/src/utils.ts diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 29751d1f42fb..894cf51cc9fa 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -1034,7 +1034,7 @@ export class Container extends EventEmitterWithErrorHandling i ? tree.trees[".protocol"].blobs.attributes : tree.blobs[".attributes"]; - const attributes = storage ? await readAndParse(storage, attributesHash) + const attributes = storage !== undefined ? await readAndParse(storage, attributesHash) : readAndParseFromBlobs(tree.trees[".protocol"].blobs, attributesHash); // Back-compat for older summaries with no term @@ -1056,7 +1056,7 @@ export class Container extends EventEmitterWithErrorHandling i if (snapshot !== undefined) { const baseTree = ".protocol" in snapshot.trees ? snapshot.trees[".protocol"] : snapshot; - if (storage) { + if (storage !== undefined) { [members, proposals, values] = await Promise.all([ readAndParse<[string, ISequencedClient][]>(storage, baseTree.blobs.quorumMembers), readAndParse<[number, ISequencedProposal, string[]][]>(storage, baseTree.blobs.quorumProposals), diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 7bcea5380356..216a0d9dc4ed 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -86,6 +86,7 @@ import { CreateChildSummarizerNodeFn, CreateChildSummarizerNodeParam, CreateSummarizerNodeSource, + ISummarizeResult, } from "@fluidframework/runtime-definitions"; import { FluidSerializer, @@ -1576,11 +1577,20 @@ export class ContainerRuntime extends EventEmitter // which it was created as it is detached container. So just use the previous snapshot. assert(this.context.baseSnapshot, "BaseSnapshot should be there as detached container loaded from snapshot"); - builder.addWithStats(key, convertSnapshotToSummaryTree(this.context.baseSnapshot.trees[key])); + const summary: ISummarizeResult = { + summary: convertSnapshotToSummaryTree(this.context.baseSnapshot.trees[key]), + stats: { + treeNodeCount: 0, + blobNodeCount: 0, + handleNodeCount: 0, + totalBlobSize: 0, + }, + }; + builder.addWithStats(key, summary); } else { const snapshot = value.generateAttachMessage().snapshot; const treeWithStats = convertToSummaryTree(snapshot, true); - builder.addWithStats(key, treeWithStats.summaryTree); + builder.addWithStats(key, treeWithStats); } }); } while (notBoundedComponentContextsLength !== this.notBoundedComponentContexts.size); diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index 1a359db5327b..fffdc3d7e8da 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -30,7 +30,7 @@ import { ChildLogger, raiseConnectedEvent, } from "@fluidframework/telemetry-utils"; -import { buildSnapshotTree } from "@fluidframework/driver-utils"; +import { buildSnapshotTree, readAndParseFromBlobs } from "@fluidframework/driver-utils"; import { TreeTreeEntry } from "@fluidframework/protocol-base"; import { IClientDetails, @@ -38,6 +38,8 @@ import { IQuorum, ISequencedDocumentMessage, ITreeEntry, + ISnapshotTree, + ITree, } from "@fluidframework/protocol-definitions"; import { IAttachMessage, @@ -51,11 +53,17 @@ import { CreateSummarizerNodeSource, } from "@fluidframework/runtime-definitions"; import { generateHandleContextPath, SummaryTreeBuilder } from "@fluidframework/runtime-utils"; -import { IChannel, IFluidDataStoreRuntime, IChannelFactory } from "@fluidframework/datastore-definitions"; +import { + IChannel, + IFluidDataStoreRuntime, + IChannelFactory, + IChannelAttributes, +} from "@fluidframework/datastore-definitions"; import { v4 as uuid } from "uuid"; import { IChannelContext, snapshotChannel } from "./channelContext"; import { LocalChannelContext } from "./localChannelContext"; import { RemoteChannelContext } from "./remoteChannelContext"; +import { convertSnapshotToITree } from "./utils"; export enum DataStoreMessageType { // Creates a new channel @@ -137,6 +145,10 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto return this._attachState; } + public get baseSnapshot(): ISnapshotTree | undefined { + return this.dataStoreContext.baseSnapshot; + } + /** * @deprecated - 0.21 back-compat */ @@ -172,6 +184,9 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto private readonly notBoundedChannelContextSet = new Set(); private boundhandles: Set | undefined; private _attachState: AttachState; + // This set stores the id of unloaded local channels. This is meant to be used only in detached container when + // loaded from a snapshot. + private readonly unLoadedLocalChannel = new Set(); private constructor( private readonly dataStoreContext: IFluidDataStoreContext, @@ -196,25 +211,44 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // Must always receive the data store type inside of the attributes if (tree?.trees !== undefined) { Object.keys(tree.trees).forEach((path) => { - const channelContext = new RemoteChannelContext( - this, - dataStoreContext, - dataStoreContext.storage, - (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), - path, - tree.trees[path], - this.sharedObjectRegistry, - undefined /* extraBlobs */, - dataStoreContext.branch, - this.dataStoreContext.summaryTracker.createOrGetChild( + let channelContext: IChannelContext; + // If already exists on storage, then create a remote channel. However, if it is loaded from a snpashot + // but not yet exists on storage, then create a Local Channel. + if (existing) { + channelContext = new RemoteChannelContext( + this, + dataStoreContext, + dataStoreContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), path, - this.deltaManager.lastSequenceNumber, - ), - this.dataStoreContext.getCreateChildSummarizerNodeFn( + tree.trees[path], + this.sharedObjectRegistry, + undefined /* extraBlobs */, + dataStoreContext.branch, + this.dataStoreContext.summaryTracker.createOrGetChild( + path, + this.deltaManager.lastSequenceNumber, + ), + this.dataStoreContext.getCreateChildSummarizerNodeFn( + path, + { type: CreateSummarizerNodeSource.FromSummary }, + )); + } else { + const channelAttributes = readAndParseFromBlobs( + tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); + channelContext = new LocalChannelContext( path, - { type: CreateSummarizerNodeSource.FromSummary }, - )); + this.sharedObjectRegistry, + channelAttributes.type, + this, + this.dataStoreContext, + this.dataStoreContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), + tree.trees[path]); + this.unLoadedLocalChannel.add(path); + } const deferred = new Deferred(); deferred.resolve(channelContext); @@ -224,7 +258,8 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } this.attachListener(); - this.bindState = existing ? BindState.Bound : BindState.NotBound; + // If exists on storage or loaded from a snapshot, it should already be binded. + this.bindState = existing || tree !== undefined ? BindState.Bound : BindState.NotBound; this._attachState = existing ? AttachState.Attached : AttachState.Detached; // If it's existing we know it has been attached. @@ -257,6 +292,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto const value = await this.contextsDeferred.get(id)!.promise; const channel = await value.getChannel(); + this.unLoadedLocalChannel.delete(channel.id); return { mimeType: "fluid/object", status: 200, value: channel }; } @@ -285,7 +321,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const context = await this.contextsDeferred.get(id)!.promise; const channel = await context.getChannel(); - + this.unLoadedLocalChannel.delete(channel.id); return channel; } @@ -302,7 +338,8 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.dataStoreContext, this.dataStoreContext.storage, (content, localOpMetadata) => this.submitChannelOp(id, content, localOpMetadata), - (address: string) => this.setChannelDirty(address)); + (address: string) => this.setChannelDirty(address), + undefined); this.contexts.set(id, context); if (this.contextsDeferred.has(id)) { @@ -314,6 +351,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.contextsDeferred.set(id, deferred); } + assert(context.channel, "Channel should be loaded when created!!"); return context.channel; } @@ -578,7 +616,16 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } if (!this.notBoundedChannelContextSet.has(objectId)) { - const snapshot = value.getAttachSnapshot(); + let snapshot: ITree; + if (this.unLoadedLocalChannel.has(objectId)) { + // If this channel is not yet loaded, then there should be no changes in the snapshot from which + // it was created as it is detached container. So just use the previous snapshot. + assert(this.dataStoreContext.baseSnapshot, + "BaseSnapshot should be there as detached container loaded from snapshot"); + snapshot = convertSnapshotToITree(this.dataStoreContext.baseSnapshot.trees[objectId]); + } else { + snapshot = value.getAttachSnapshot(); + } // And then store the tree entries.push(new TreeTreeEntry(objectId, snapshot)); diff --git a/packages/runtime/datastore/src/localChannelContext.ts b/packages/runtime/datastore/src/localChannelContext.ts index 1fe119a1cfc4..84044ae3839b 100644 --- a/packages/runtime/datastore/src/localChannelContext.ts +++ b/packages/runtime/datastore/src/localChannelContext.ts @@ -8,69 +8,89 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions"; import { ISequencedDocumentMessage, ITree, + ISnapshotTree, } from "@fluidframework/protocol-definitions"; -import { IChannel, IFluidDataStoreRuntime } from "@fluidframework/datastore-definitions"; +import { + IChannel, + IFluidDataStoreRuntime, + IChannelFactory, + IChannelAttributes, +} from "@fluidframework/datastore-definitions"; import { IFluidDataStoreContext, ISummarizeResult } from "@fluidframework/runtime-definitions"; +import { readAndParse } from "@fluidframework/driver-utils"; +import { CreateContainerError } from "@fluidframework/container-utils"; import { convertToSummaryTree } from "@fluidframework/runtime-utils"; import { createServiceEndpoints, IChannelContext, snapshotChannel } from "./channelContext"; import { ChannelDeltaConnection } from "./channelDeltaConnection"; import { ISharedObjectRegistry } from "./dataStoreRuntime"; +import { ChannelStorageService } from "./channelStorageService"; /** * Channel context for a locally created channel */ export class LocalChannelContext implements IChannelContext { - public readonly channel: IChannel; + public channel: IChannel | undefined; + private isLoaded = false; private attached = false; - private connection: ChannelDeltaConnection | undefined; + private readonly pending: ISequencedDocumentMessage[] = []; + private _services: { + readonly deltaConnection: ChannelDeltaConnection, + readonly objectStorage: ChannelStorageService, + } | undefined; private readonly dirtyFn: () => void; + private readonly factory: IChannelFactory | undefined; constructor( - id: string, + private readonly id: string, registry: ISharedObjectRegistry, type: string, - runtime: IFluidDataStoreRuntime, - private readonly dataStoreContext: IFluidDataStoreContext, + private readonly runtime: IFluidDataStoreRuntime, + private readonly componentContext: IFluidDataStoreContext, private readonly storageService: IDocumentStorageService, private readonly submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: (address: string) => void, + private readonly snapshotTree: ISnapshotTree | undefined, ) { - const factory = registry.get(type); - if (factory === undefined) { + this.factory = registry.get(type); + if (this.factory === undefined) { throw new Error(`Channel Factory ${type} not registered`); } - - this.channel = factory.create(runtime, id); - + if (snapshotTree === undefined) { + this.channel = this.factory.create(runtime, id); + this.isLoaded = true; + } this.dirtyFn = () => { dirtyFn(id); }; } public async getChannel(): Promise { + if (this.channel === undefined) { + this.channel = await this.loadChannel(); + } return this.channel; } public setConnectionState(connected: boolean, clientId?: string) { - // Connection events are ignored if the data store is not yet attached + // Connection events are ignored if the component is not yet attached if (!this.attached) { return; } - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.setConnectionState(connected); + this.services.deltaConnection.setConnectionState(connected); } public processOp(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void { assert(this.attached, "Local channel must be attached when processing op"); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.process(message, local, localOpMetadata); + if (this.isLoaded) { + this.services.deltaConnection.process(message, local, localOpMetadata); + } else { + this.pending.push(message); + } } public reSubmit(content: any, localOpMetadata: unknown) { + assert(this.isLoaded, "Channel should be loaded to resubmit ops"); assert(this.attached, "Local channel must be attached when resubmitting op"); - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.connection!.reSubmit(content, localOpMetadata); + this.services.deltaConnection.reSubmit(content, localOpMetadata); } public async snapshot(fullTree: boolean = false): Promise { @@ -84,23 +104,94 @@ export class LocalChannelContext implements IChannelContext { } public getAttachSnapshot(): ITree { + assert(this.isLoaded && this.channel !== undefined, "Channel should be loaded to take snapshot"); return snapshotChannel(this.channel); } + private async loadChannel(): Promise { + assert(!this.isLoaded, "Channel must not already be loaded when loading"); + assert(this.snapshotTree, "Snapshot should be provided to load from!!"); + + assert(await this.services.objectStorage.contains(".attributes"), ".attributes blob should be present"); + const attributes = await readAndParse( + this.services.objectStorage, + ".attributes"); + + assert(this.factory, "Factory should be there for local channel"); + const channel = await this.factory.loadLocal( + this.runtime, + this.id, + this.services.objectStorage, + attributes); + + // Commit changes. + this.channel = channel; + this.isLoaded = true; + + if (this.attached) { + this.channel.connect(this.services); + } + + // Send all pending messages to the channel + for (const message of this.pending) { + try { + this.services.deltaConnection.process(message, false, undefined /* localOpMetadata */); + } catch (err) { + // record sequence number for easier debugging + const error = CreateContainerError(err); + error.sequenceNumber = message.sequenceNumber; + throw error; + } + } + return this.channel; + } + public attach(): void { if (this.attached) { throw new Error("Channel is already attached"); } - const services = createServiceEndpoints( - this.channel.id, - this.dataStoreContext.connected, - this.submitFn, - this.dirtyFn, - this.storageService); - this.connection = services.deltaConnection; - this.channel.connect(services); - + if (this.isLoaded) { + assert(this.channel, "Channel should be there if loaded!!"); + this.channel.connect(this.services); + } this.attached = true; } + + private get services() { + if (this._services === undefined) { + let blobMap: Map | undefined; + if (this.snapshotTree !== undefined) { + blobMap = new Map(); + this.collectExtraBlobsAndSanitizeSnapshot(this.snapshotTree, blobMap); + } + this._services = createServiceEndpoints( + this.id, + this.componentContext.connected, + this.submitFn, + this.dirtyFn, + this.storageService, + this.snapshotTree !== undefined ? Promise.resolve(this.snapshotTree) : undefined, + blobMap !== undefined ? + Promise.resolve(blobMap) : undefined, + ); + } + return this._services; + } + + private collectExtraBlobsAndSanitizeSnapshot(snapshotTree: ISnapshotTree, blobMap: Map) { + const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); + for (const [blobName, blobId] of blobMapInitial.entries()) { + const blobValue = blobMapInitial.get(blobId); + if (blobValue !== undefined) { + blobMap.set(blobId, blobValue); + } else { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete snapshotTree.blobs[blobName]; + } + } + for (const value of Object.values(snapshotTree.trees)) { + this.collectExtraBlobsAndSanitizeSnapshot(value, blobMap); + } + } } diff --git a/packages/runtime/datastore/src/utils.ts b/packages/runtime/datastore/src/utils.ts new file mode 100644 index 000000000000..4ca462965d7c --- /dev/null +++ b/packages/runtime/datastore/src/utils.ts @@ -0,0 +1,32 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +import { Buffer } from "buffer"; +import { ISnapshotTree, ITreeEntry, ITree } from "@fluidframework/protocol-definitions"; +import { BlobTreeEntry, TreeTreeEntry } from "@fluidframework/protocol-base"; + +export function convertSnapshotToITree(snapshotTree: ISnapshotTree): ITree { + const entries: ITreeEntry[] = []; + const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); + const blobMapFinal = new Map(); + for (const [key, value] of blobMapInitial.entries()) { + if (blobMapInitial.has(value)) { + blobMapFinal[key] = blobMapInitial.get(value); + } + } + for (const [key, value] of Object.entries(blobMapFinal)) { + const decoded = Buffer.from(value, "base64").toString(); + entries.push(new BlobTreeEntry(key, decoded)); + } + for (const [key, tree] of Object.entries(snapshotTree.trees)) { + entries.push(new TreeTreeEntry(key, convertSnapshotToITree(tree))); + } + const finalTree: ITree = { + entries, + // eslint-disable-next-line no-null/no-null + id: null, + }; + return finalTree; +} From d240a3a12b23555d33852d1b267116981804f81e Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 21:51:21 -0700 Subject: [PATCH 04/27] remove --- .../component-runtime/src/channelContext.ts | 4 +- .../component-runtime/src/componentRuntime.ts | 95 +++--------- .../src/localChannelContext.ts | 145 ++++-------------- .../runtime/component-runtime/src/utils.ts | 32 ---- 4 files changed, 53 insertions(+), 223 deletions(-) delete mode 100644 packages/runtime/component-runtime/src/utils.ts diff --git a/packages/runtime/component-runtime/src/channelContext.ts b/packages/runtime/component-runtime/src/channelContext.ts index 63c543c568d2..ff69a7c1125f 100644 --- a/packages/runtime/component-runtime/src/channelContext.ts +++ b/packages/runtime/component-runtime/src/channelContext.ts @@ -36,8 +36,8 @@ export function createServiceEndpoints( submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: () => void, storageService: IDocumentStorageService, - tree?: Promise | undefined, - extraBlobs?: Promise> | undefined, + tree?: Promise, + extraBlobs?: Promise>, ) { const deltaConnection = new ChannelDeltaConnection( id, diff --git a/packages/runtime/component-runtime/src/componentRuntime.ts b/packages/runtime/component-runtime/src/componentRuntime.ts index 4887a62e02b8..498840cca2dc 100644 --- a/packages/runtime/component-runtime/src/componentRuntime.ts +++ b/packages/runtime/component-runtime/src/componentRuntime.ts @@ -30,7 +30,7 @@ import { ChildLogger, raiseConnectedEvent, } from "@fluidframework/telemetry-utils"; -import { buildSnapshotTree, readAndParseFromBlobs } from "@fluidframework/driver-utils"; +import { buildSnapshotTree } from "@fluidframework/driver-utils"; import { TreeTreeEntry } from "@fluidframework/protocol-base"; import { IClientDetails, @@ -38,7 +38,6 @@ import { IQuorum, ISequencedDocumentMessage, ITreeEntry, - ITree, } from "@fluidframework/protocol-definitions"; import { IAttachMessage, @@ -52,17 +51,11 @@ import { CreateSummarizerNodeSource, } from "@fluidframework/runtime-definitions"; import { generateHandleContextPath, SummaryTreeBuilder } from "@fluidframework/runtime-utils"; -import { - IChannel, - IFluidDataStoreRuntime, - IChannelFactory, - IChannelAttributes, -} from "@fluidframework/datastore-definitions"; +import { IChannel, IFluidDataStoreRuntime, IChannelFactory } from "@fluidframework/datastore-definitions"; import { v4 as uuid } from "uuid"; import { IChannelContext, snapshotChannel } from "./channelContext"; import { LocalChannelContext } from "./localChannelContext"; import { RemoteChannelContext } from "./remoteChannelContext"; -import { convertSnapshotToITree } from "./utils"; export enum ComponentMessageType { // Creates a new channel @@ -179,9 +172,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto private readonly notBoundedChannelContextSet = new Set(); private boundhandles: Set | undefined; private _attachState: AttachState; - // This set stores the id of unloaded local channels. This is meant to be used only in detached container when - // loaded from a snapshot. - private readonly unLoadedLocalChannel = new Set(); private constructor( private readonly componentContext: IFluidDataStoreContext, @@ -206,45 +196,25 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // Must always receive the component type inside of the attributes if (tree?.trees !== undefined) { Object.keys(tree.trees).forEach((path) => { - let channelContext: IChannelContext; - // If already exists on storage, then create a remote channel. However, if it is loaded from a snpashot - // but not yet exists on storage, then create a Local Channel. - if (existing) { - channelContext = new RemoteChannelContext( - this, - componentContext, - componentContext.storage, - (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), + const channelContext = new RemoteChannelContext( + this, + componentContext, + componentContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), + path, + tree.trees[path], + this.sharedObjectRegistry, + undefined /* extraBlobs */, + componentContext.branch, + this.componentContext.summaryTracker.createOrGetChild( path, - tree.trees[path], - this.sharedObjectRegistry, - undefined /* extraBlobs */, - componentContext.branch, - this.componentContext.summaryTracker.createOrGetChild( - path, - this.deltaManager.lastSequenceNumber, - )); - } else { - const channelAttributes = readAndParseFromBlobs( - tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); - channelContext = new LocalChannelContext( + this.deltaManager.lastSequenceNumber, + ), + this.componentContext.getCreateChildSummarizerNodeFn( path, - this.sharedObjectRegistry, - channelAttributes.type, - this, - this.componentContext, - this.componentContext.storage, - (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), - tree.trees[path]); - this.unLoadedLocalChannel.add(path); - } - - this.componentContext.getCreateChildSummarizerNodeFn( - path, - { type: CreateSummarizerNodeSource.FromSummary }, - )); + { type: CreateSummarizerNodeSource.FromSummary }, + )); const deferred = new Deferred(); deferred.resolve(channelContext); @@ -254,9 +224,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } this.attachListener(); - - // If exists on storage or loaded from a snapshot, it should already be binded. - this.bindState = existing || tree !== undefined ? BindState.Bound : BindState.NotBound; + this.bindState = existing ? BindState.Bound : BindState.NotBound; this._attachState = existing ? AttachState.Attached : AttachState.Detached; // If it's existing we know it has been attached. @@ -265,10 +233,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } } - public get baseSnapshot() { - return this.componentContext.baseSnapshot; - } - public dispose(): void { if (this._disposed) { return; @@ -292,7 +256,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const value = await this.contextsDeferred.get(id)!.promise; const channel = await value.getChannel(); - this.unLoadedLocalChannel.delete(channel.id); + return { mimeType: "fluid/object", status: 200, value: channel }; } @@ -321,7 +285,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const context = await this.contextsDeferred.get(id)!.promise; const channel = await context.getChannel(); - this.unLoadedLocalChannel.delete(channel.id); + return channel; } @@ -338,8 +302,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.componentContext, this.componentContext.storage, (content, localOpMetadata) => this.submitChannelOp(id, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), - undefined); + (address: string) => this.setChannelDirty(address)); this.contexts.set(id, context); if (this.contextsDeferred.has(id)) { @@ -351,7 +314,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.contextsDeferred.set(id, deferred); } - assert(context.channel, "Channel should be loaded when created!!"); return context.channel; } @@ -616,16 +578,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto } if (!this.notBoundedChannelContextSet.has(objectId)) { - let snapshot: ITree; - if (this.unLoadedLocalChannel.has(objectId)) { - // If this channel is not yet loaded, then there should be no changes in the snapshot from which - // it was created as it is detached container. So just use the previous snapshot. - assert(this.componentContext.baseSnapshot, - "BaseSnapshot should be there as detached container loaded from snapshot"); - snapshot = convertSnapshotToITree(this.componentContext.baseSnapshot.trees[objectId]); - } else { - snapshot = value.getAttachSnapshot(); - } + const snapshot = value.getAttachSnapshot(); // And then store the tree entries.push(new TreeTreeEntry(objectId, snapshot)); diff --git a/packages/runtime/component-runtime/src/localChannelContext.ts b/packages/runtime/component-runtime/src/localChannelContext.ts index 0fe84eacb793..514bf8fca73b 100644 --- a/packages/runtime/component-runtime/src/localChannelContext.ts +++ b/packages/runtime/component-runtime/src/localChannelContext.ts @@ -8,64 +8,44 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions"; import { ISequencedDocumentMessage, ITree, - ISnapshotTree, } from "@fluidframework/protocol-definitions"; -import { - IChannel, - IFluidDataStoreRuntime, - IChannelFactory, - IChannelAttributes, -} from "@fluidframework/datastore-definitions""; +import { IChannel, IFluidDataStoreRuntime } from "@fluidframework/datastore-definitions"; import { IFluidDataStoreContext, ISummarizeResult } from "@fluidframework/runtime-definitions"; -import { readAndParse } from "@fluidframework/driver-utils"; -import { CreateContainerError } from "@fluidframework/container-utils"; import { convertToSummaryTree } from "@fluidframework/runtime-utils"; import { createServiceEndpoints, IChannelContext, snapshotChannel } from "./channelContext"; import { ChannelDeltaConnection } from "./channelDeltaConnection"; import { ISharedObjectRegistry } from "./componentRuntime"; -import { ChannelStorageService } from "./channelStorageService"; /** * Channel context for a locally created channel */ export class LocalChannelContext implements IChannelContext { - public channel: IChannel | undefined; - private isLoaded = false; + public readonly channel: IChannel; private attached = false; - private readonly pending: ISequencedDocumentMessage[] = []; - private _services: { - readonly deltaConnection: ChannelDeltaConnection, - readonly objectStorage: ChannelStorageService, - } | undefined; + private connection: ChannelDeltaConnection | undefined; private readonly dirtyFn: () => void; - private readonly factory: IChannelFactory | undefined; constructor( - private readonly id: string, + id: string, registry: ISharedObjectRegistry, type: string, - private readonly runtime: IFluidDataStoreRuntime, + runtime: IFluidDataStoreRuntime, private readonly componentContext: IFluidDataStoreContext, private readonly storageService: IDocumentStorageService, private readonly submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: (address: string) => void, - private readonly snapshotTree: ISnapshotTree | undefined, ) { - this.factory = registry.get(type); - if (this.factory === undefined) { + const factory = registry.get(type); + if (factory === undefined) { throw new Error(`Channel Factory ${type} not registered`); } - if (snapshotTree === undefined) { - this.channel = this.factory.create(runtime, id); - this.isLoaded = true; - } + + this.channel = factory.create(runtime, id); + this.dirtyFn = () => { dirtyFn(id); }; } public async getChannel(): Promise { - if (this.channel === undefined) { - this.channel = await this.loadChannel(); - } return this.channel; } @@ -74,23 +54,23 @@ export class LocalChannelContext implements IChannelContext { if (!this.attached) { return; } - this.services.deltaConnection.setConnectionState(connected); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.connection!.setConnectionState(connected); } public processOp(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void { assert(this.attached, "Local channel must be attached when processing op"); - if (this.isLoaded) { - this.services.deltaConnection.process(message, local, localOpMetadata); - } else { - this.pending.push(message); - } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.connection!.process(message, local, localOpMetadata); } public reSubmit(content: any, localOpMetadata: unknown) { - assert(this.isLoaded, "Channel should be loaded to resubmit ops"); assert(this.attached, "Local channel must be attached when resubmitting op"); - this.services.deltaConnection.reSubmit(content, localOpMetadata); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.connection!.reSubmit(content, localOpMetadata); } public async snapshot(fullTree: boolean = false): Promise { @@ -104,94 +84,23 @@ export class LocalChannelContext implements IChannelContext { } public getAttachSnapshot(): ITree { - assert(this.isLoaded && this.channel !== undefined, "Channel should be loaded to take snapshot"); return snapshotChannel(this.channel); } - private async loadChannel(): Promise { - assert(!this.isLoaded, "Channel must not already be loaded when loading"); - assert(this.snapshotTree, "Snapshot should be provided to load from!!"); - - assert(await this.services.objectStorage.contains(".attributes"), ".attributes blob should be present"); - const attributes = await readAndParse( - this.services.objectStorage, - ".attributes"); - - assert(this.factory, "Factory should be there for local channel"); - const channel = await this.factory.loadLocal( - this.runtime, - this.id, - this.services.objectStorage, - attributes); - - // Commit changes. - this.channel = channel; - this.isLoaded = true; - - if (this.attached) { - this.channel.connect(this.services); - } - - // Send all pending messages to the channel - for (const message of this.pending) { - try { - this.services.deltaConnection.process(message, false, undefined /* localOpMetadata */); - } catch (err) { - // record sequence number for easier debugging - const error = CreateContainerError(err); - error.sequenceNumber = message.sequenceNumber; - throw error; - } - } - return this.channel; - } - public attach(): void { if (this.attached) { throw new Error("Channel is already attached"); } - if (this.isLoaded) { - assert(this.channel, "Channel should be there if loaded!!"); - this.channel.connect(this.services); - } - this.attached = true; - } - - private get services() { - if (this._services === undefined) { - let blobMap: Map | undefined; - if (this.snapshotTree !== undefined) { - blobMap = new Map(); - this.collectExtraBlobsAndSanitizeSnapshot(this.snapshotTree, blobMap); - } - this._services = createServiceEndpoints( - this.id, - this.componentContext.connected, - this.submitFn, - this.dirtyFn, - this.storageService, - this.snapshotTree !== undefined ? Promise.resolve(this.snapshotTree) : undefined, - blobMap !== undefined ? - Promise.resolve(blobMap) : undefined, - ); - } - return this._services; - } + const services = createServiceEndpoints( + this.channel.id, + this.componentContext.connected, + this.submitFn, + this.dirtyFn, + this.storageService); + this.connection = services.deltaConnection; + this.channel.connect(services); - private collectExtraBlobsAndSanitizeSnapshot(snapshotTree: ISnapshotTree, blobMap: Map) { - const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); - for (const [blobName, blobId] of blobMapInitial.entries()) { - const blobValue = blobMapInitial.get(blobId); - if (blobValue !== undefined) { - blobMap.set(blobId, blobValue); - } else { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete snapshotTree.blobs[blobName]; - } - } - for (const value of Object.values(snapshotTree.trees)) { - this.collectExtraBlobsAndSanitizeSnapshot(value, blobMap); - } + this.attached = true; } } diff --git a/packages/runtime/component-runtime/src/utils.ts b/packages/runtime/component-runtime/src/utils.ts deleted file mode 100644 index 4ca462965d7c..000000000000 --- a/packages/runtime/component-runtime/src/utils.ts +++ /dev/null @@ -1,32 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import { Buffer } from "buffer"; -import { ISnapshotTree, ITreeEntry, ITree } from "@fluidframework/protocol-definitions"; -import { BlobTreeEntry, TreeTreeEntry } from "@fluidframework/protocol-base"; - -export function convertSnapshotToITree(snapshotTree: ISnapshotTree): ITree { - const entries: ITreeEntry[] = []; - const blobMapInitial = new Map(Object.entries(snapshotTree.blobs)); - const blobMapFinal = new Map(); - for (const [key, value] of blobMapInitial.entries()) { - if (blobMapInitial.has(value)) { - blobMapFinal[key] = blobMapInitial.get(value); - } - } - for (const [key, value] of Object.entries(blobMapFinal)) { - const decoded = Buffer.from(value, "base64").toString(); - entries.push(new BlobTreeEntry(key, decoded)); - } - for (const [key, tree] of Object.entries(snapshotTree.trees)) { - entries.push(new TreeTreeEntry(key, convertSnapshotToITree(tree))); - } - const finalTree: ITree = { - entries, - // eslint-disable-next-line no-null/no-null - id: null, - }; - return finalTree; -} From 217aaccfcc79cc7b2ffc290f68fbeeb4e3bf859e Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 22:00:47 -0700 Subject: [PATCH 05/27] fix tiny --- .../src/getTinyliciousContainer.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts b/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts index 3bdf2dffe96e..a3d32a1ed107 100644 --- a/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts +++ b/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts @@ -5,7 +5,7 @@ import { IRequest } from "@fluidframework/component-core-interfaces"; import { - IRuntimeFactory, + IRuntimeFactory, IDetachedContainerSource, } from "@fluidframework/container-definitions"; import { Container, Loader } from "@fluidframework/container-loader"; import { @@ -100,7 +100,11 @@ export async function getTinyliciousContainer( // We're not actually using the code proposal (our code loader always loads the same module regardless of the // proposal), but the Container will only give us a NullRuntime if there's no proposal. So we'll use a fake // proposal. - container = await loader.createDetachedContainer({ package: "", config: {} }); + const source: IDetachedContainerSource = { + codeDetails: { package: "", config: {} }, + useSnapshot: false, + }; + container = await loader.createDetachedContainer(source); await container.attach({ url: documentId }); } else { // The InsecureTinyliciousUrlResolver expects the url of the request to be the documentId. From 1bb9da0590313c2ea41ea4b2a8a80bb1e84da14d Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 22:13:36 -0700 Subject: [PATCH 06/27] remove unloaded map --- packages/runtime/datastore/src/dataStoreRuntime.ts | 13 ++++--------- .../runtime/datastore/src/localChannelContext.ts | 10 +++++++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index fffdc3d7e8da..ce847d046868 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -184,9 +184,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto private readonly notBoundedChannelContextSet = new Set(); private boundhandles: Set | undefined; private _attachState: AttachState; - // This set stores the id of unloaded local channels. This is meant to be used only in detached container when - // loaded from a snapshot. - private readonly unLoadedLocalChannel = new Set(); private constructor( private readonly dataStoreContext: IFluidDataStoreContext, @@ -247,7 +244,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), (address: string) => this.setChannelDirty(address), tree.trees[path]); - this.unLoadedLocalChannel.add(path); } const deferred = new Deferred(); deferred.resolve(channelContext); @@ -292,7 +288,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto const value = await this.contextsDeferred.get(id)!.promise; const channel = await value.getChannel(); - this.unLoadedLocalChannel.delete(channel.id); return { mimeType: "fluid/object", status: 200, value: channel }; } @@ -321,7 +316,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const context = await this.contextsDeferred.get(id)!.promise; const channel = await context.getChannel(); - this.unLoadedLocalChannel.delete(channel.id); + return channel; } @@ -617,14 +612,14 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto if (!this.notBoundedChannelContextSet.has(objectId)) { let snapshot: ITree; - if (this.unLoadedLocalChannel.has(objectId)) { + if (value.isLoaded) { + snapshot = value.getAttachSnapshot(); + } else { // If this channel is not yet loaded, then there should be no changes in the snapshot from which // it was created as it is detached container. So just use the previous snapshot. assert(this.dataStoreContext.baseSnapshot, "BaseSnapshot should be there as detached container loaded from snapshot"); snapshot = convertSnapshotToITree(this.dataStoreContext.baseSnapshot.trees[objectId]); - } else { - snapshot = value.getAttachSnapshot(); } // And then store the tree diff --git a/packages/runtime/datastore/src/localChannelContext.ts b/packages/runtime/datastore/src/localChannelContext.ts index 84044ae3839b..31bbb414536f 100644 --- a/packages/runtime/datastore/src/localChannelContext.ts +++ b/packages/runtime/datastore/src/localChannelContext.ts @@ -30,7 +30,7 @@ import { ChannelStorageService } from "./channelStorageService"; */ export class LocalChannelContext implements IChannelContext { public channel: IChannel | undefined; - private isLoaded = false; + private _isLoaded = false; private attached = false; private readonly pending: ISequencedDocumentMessage[] = []; private _services: { @@ -57,7 +57,7 @@ export class LocalChannelContext implements IChannelContext { } if (snapshotTree === undefined) { this.channel = this.factory.create(runtime, id); - this.isLoaded = true; + this._isLoaded = true; } this.dirtyFn = () => { dirtyFn(id); }; } @@ -69,6 +69,10 @@ export class LocalChannelContext implements IChannelContext { return this.channel; } + public get isLoaded(): boolean { + return this._isLoaded; + } + public setConnectionState(connected: boolean, clientId?: string) { // Connection events are ignored if the component is not yet attached if (!this.attached) { @@ -126,7 +130,7 @@ export class LocalChannelContext implements IChannelContext { // Commit changes. this.channel = channel; - this.isLoaded = true; + this._isLoaded = true; if (this.attached) { this.channel.connect(this.services); From 6fcce12172bc6b836d15fa7fdf26618418356465 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 22:21:12 -0700 Subject: [PATCH 07/27] remove unrealized map --- .../container-runtime/src/componentContext.ts | 4 ++++ .../container-runtime/src/containerRuntime.ts | 15 +++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/runtime/container-runtime/src/componentContext.ts b/packages/runtime/container-runtime/src/componentContext.ts index b616759c3f3b..3119d6c30653 100644 --- a/packages/runtime/container-runtime/src/componentContext.ts +++ b/packages/runtime/container-runtime/src/componentContext.ts @@ -141,6 +141,10 @@ export abstract class FluidDataStoreContext extends EventEmitter implements return this._containerRuntime; } + public get isLoaded(): boolean { + return this.loaded; + } + /** * @deprecated 0.17 Issue #1888 Rename IHostRuntime to IContainerRuntime and refactor usages * Use containerRuntime instead of hostRuntime diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 216a0d9dc4ed..32288d59bbda 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -606,9 +606,6 @@ export class ContainerRuntime extends EventEmitter ) => CreateChildSummarizerNodeFn; } & ({ readonly enabled: true; readonly node: SummarizerNode } | { readonly enabled: false }); private readonly notBoundedComponentContexts = new Set(); - // This set stores the id of unrealized components. This is meant to be used only in detached container when loaded - // from a snapshot. - private readonly unRealizedComponents = new Set(); private tasks: string[] = []; @@ -772,7 +769,6 @@ export class ContainerRuntime extends EventEmitter } else { throw new Error(`Invalid snapshot format version ${snapshotFormatVersion}`); } - this.unRealizedComponents.add(key); componentContext = new LocalFluidDataStoreContext( key, pkgFromSnapshot, @@ -1172,7 +1168,6 @@ export class ContainerRuntime extends EventEmitter } const componentContext = await deferredContext.promise; - this.unRealizedComponents.delete(id); return componentContext.realize(); } @@ -1572,7 +1567,11 @@ export class ContainerRuntime extends EventEmitter !(this.notBoundedComponentContexts.has(key) || builderTree[key]), ) .map(([key, value]) => { - if (this.unRealizedComponents.has(key)) { + if (value.isLoaded) { + const snapshot = value.generateAttachMessage().snapshot; + const treeWithStats = convertToSummaryTree(snapshot, true); + builder.addWithStats(key, treeWithStats); + } else { // If this component is not yet loaded, then there should be no changes in the snapshot from // which it was created as it is detached container. So just use the previous snapshot. assert(this.context.baseSnapshot, @@ -1587,10 +1586,6 @@ export class ContainerRuntime extends EventEmitter }, }; builder.addWithStats(key, summary); - } else { - const snapshot = value.generateAttachMessage().snapshot; - const treeWithStats = convertToSummaryTree(snapshot, true); - builder.addWithStats(key, treeWithStats); } }); } while (notBoundedComponentContextsLength !== this.notBoundedComponentContexts.size); From 3f903b7cc8c81ee35aa3fc3725768a22cf780edc Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 22:41:13 -0700 Subject: [PATCH 08/27] add comment --- packages/runtime/container-runtime/src/containerRuntime.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 32288d59bbda..d0d14af50c02 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -1578,6 +1578,7 @@ export class ContainerRuntime extends EventEmitter "BaseSnapshot should be there as detached container loaded from snapshot"); const summary: ISummarizeResult = { summary: convertSnapshotToSummaryTree(this.context.baseSnapshot.trees[key]), + // No need to build stats, because we don't need them anyway here. stats: { treeNodeCount: 0, blobNodeCount: 0, From e255f543d328c737f4b987387917876cf3b491ae Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 4 Aug 2020 23:43:24 -0700 Subject: [PATCH 09/27] remove loadlocal --- packages/dds/cell/src/cellFactory.ts | 14 +- packages/dds/counter/src/counterFactory.ts | 12 -- packages/dds/ink/src/inkFactory.ts | 15 -- packages/dds/map/src/directory.ts | 15 -- packages/dds/map/src/map.ts | 15 -- packages/dds/matrix/src/runtime.ts | 12 -- .../src/consensusOrderedCollectionFactory.ts | 12 -- .../src/consensusRegisterCollectionFactory.ts | 12 -- packages/dds/sequence/src/sequenceFactory.ts | 34 ---- .../sequence/src/sharedIntervalCollection.ts | 12 -- packages/dds/sequence/src/sparsematrix.ts | 12 -- .../shared-object-base/src/sharedObject.ts | 26 +-- .../src/sharedSummaryBlockFactory.ts | 16 -- .../src/channel.ts | 187 ++++++++++++++++++ .../datastore-definitions/src/channel.ts | 19 +- .../datastore/src/localChannelContext.ts | 5 +- 16 files changed, 201 insertions(+), 217 deletions(-) create mode 100644 packages/runtime/component-runtime-definitions/src/channel.ts diff --git a/packages/dds/cell/src/cellFactory.ts b/packages/dds/cell/src/cellFactory.ts index 9b7d9cdc64d4..ea45dbcaafba 100644 --- a/packages/dds/cell/src/cellFactory.ts +++ b/packages/dds/cell/src/cellFactory.ts @@ -8,7 +8,6 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { SharedCell } from "./cell"; import { ISharedCell } from "./interfaces"; @@ -38,24 +37,13 @@ export class CellFactory implements IChannelFactory { runtime: IFluidDataStoreRuntime, id: string, services: IChannelServices, - branchId: string, + branchId: string | undefined, attributes: IChannelAttributes): Promise { const cell = new SharedCell(id, runtime, attributes); await cell.load(branchId, services); return cell; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const cell = new SharedCell(id, runtime, attributes); - await cell.loadLocal(objectStorage); - return cell; - } - public create(document: IFluidDataStoreRuntime, id: string): ISharedCell { const cell = new SharedCell(id, document, this.attributes); cell.initializeLocal(); diff --git a/packages/dds/counter/src/counterFactory.ts b/packages/dds/counter/src/counterFactory.ts index 4ceeeaadb00d..b2f50174f13e 100644 --- a/packages/dds/counter/src/counterFactory.ts +++ b/packages/dds/counter/src/counterFactory.ts @@ -8,7 +8,6 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { SharedCounter } from "./counter"; import { ISharedCounter } from "./interfaces"; @@ -45,17 +44,6 @@ export class CounterFactory implements IChannelFactory { return counter; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const counter = new SharedCounter(id, runtime, attributes); - await counter.loadLocal(objectStorage); - return counter; - } - public create(document: IFluidDataStoreRuntime, id: string): ISharedCounter { const counter = new SharedCounter(id, document, this.attributes); counter.initializeLocal(); diff --git a/packages/dds/ink/src/inkFactory.ts b/packages/dds/ink/src/inkFactory.ts index bed731b4fe4f..e4b390be90dc 100644 --- a/packages/dds/ink/src/inkFactory.ts +++ b/packages/dds/ink/src/inkFactory.ts @@ -8,7 +8,6 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ISharedObject } from "@fluidframework/shared-object-base"; import { Ink } from "./ink"; @@ -62,20 +61,6 @@ export class InkFactory implements IChannelFactory { return ink; } - /** - * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} - */ - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const ink = new Ink(runtime, id, attributes); - await ink.loadLocal(objectStorage); - return ink; - } - /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/dds/map/src/directory.ts b/packages/dds/map/src/directory.ts index 52cc9bf85953..5f8b6fd7b7df 100644 --- a/packages/dds/map/src/directory.ts +++ b/packages/dds/map/src/directory.ts @@ -343,21 +343,6 @@ export class DirectoryFactory { return directory; } - /** - * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} - */ - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const directory = new SharedDirectory(id, runtime, attributes); - await directory.loadLocal(objectStorage); - - return directory; - } - /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/dds/map/src/map.ts b/packages/dds/map/src/map.ts index 88727c38dfd2..aed88714aad4 100644 --- a/packages/dds/map/src/map.ts +++ b/packages/dds/map/src/map.ts @@ -86,21 +86,6 @@ export class MapFactory implements IChannelFactory { return map; } - /** - * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} - */ - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const map = new SharedMap(id, runtime, attributes); - await map.loadLocal(objectStorage); - - return map; - } - /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/dds/matrix/src/runtime.ts b/packages/dds/matrix/src/runtime.ts index ecc1b1d20320..bb13d57c2469 100644 --- a/packages/dds/matrix/src/runtime.ts +++ b/packages/dds/matrix/src/runtime.ts @@ -9,7 +9,6 @@ import { IChannelServices, IChannel, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { pkgVersion } from "./packageVersion"; import { SharedMatrix } from "./matrix"; @@ -43,17 +42,6 @@ export class SharedMatrixFactory implements IChannelFactory { return matrix; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const matrix = new SharedMatrix(runtime, id, attributes); - await matrix.loadLocal(objectStorage); - return matrix; - } - public create(document: IFluidDataStoreRuntime, id: string): IChannel { const matrix = new SharedMatrix(document, id, this.attributes); matrix.initializeLocal(); diff --git a/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts b/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts index 59f85f146530..e3a2669a78c8 100644 --- a/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts +++ b/packages/dds/ordered-collection/src/consensusOrderedCollectionFactory.ts @@ -7,7 +7,6 @@ import { IChannelAttributes, IFluidDataStoreRuntime, IChannelServices, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ConsensusQueue } from "./consensusQueue"; import { IConsensusOrderedCollection, IConsensusOrderedCollectionFactory } from "./interfaces"; @@ -44,17 +43,6 @@ export class ConsensusQueueFactory implements IConsensusOrderedCollectionFactory return collection; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const collection = new ConsensusQueue(id, runtime, attributes); - await collection.loadLocal(objectStorage); - return collection; - } - public create(document: IFluidDataStoreRuntime, id: string): IConsensusOrderedCollection { const collection = new ConsensusQueue(id, document, this.attributes); collection.initializeLocal(); diff --git a/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts b/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts index a77b52b0f066..93efd6071b14 100644 --- a/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts +++ b/packages/dds/register-collection/src/consensusRegisterCollectionFactory.ts @@ -7,7 +7,6 @@ import { IChannelAttributes, IFluidDataStoreRuntime, IChannelServices, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ConsensusRegisterCollection } from "./consensusRegisterCollection"; import { IConsensusRegisterCollection, IConsensusRegisterCollectionFactory } from "./interfaces"; @@ -44,17 +43,6 @@ export class ConsensusRegisterCollectionFactory implements IConsensusRegisterCol return collection; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const collection = new ConsensusRegisterCollection(id, runtime, attributes); - await collection.loadLocal(objectStorage); - return collection; - } - public create(document: IFluidDataStoreRuntime, id: string): IConsensusRegisterCollection { const collection = new ConsensusRegisterCollection(id, document, ConsensusRegisterCollectionFactory.Attributes); collection.initializeLocal(); diff --git a/packages/dds/sequence/src/sequenceFactory.ts b/packages/dds/sequence/src/sequenceFactory.ts index 8fffd5e1e5c4..3968243a8e6d 100644 --- a/packages/dds/sequence/src/sequenceFactory.ts +++ b/packages/dds/sequence/src/sequenceFactory.ts @@ -9,7 +9,6 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ISharedObject } from "@fluidframework/shared-object-base"; import { pkgVersion } from "./packageVersion"; @@ -56,17 +55,6 @@ export class SharedStringFactory implements IChannelFactory { return sharedString; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const sharedString = new SharedString(runtime, id, attributes); - await sharedString.loadLocal(objectStorage); - return sharedString; - } - public create(document: IFluidDataStoreRuntime, id: string): SharedString { const sharedString = new SharedString(document, id, this.attributes); sharedString.initializeLocal(); @@ -113,17 +101,6 @@ export class SharedObjectSequenceFactory implements IChannelFactory { return sharedSeq; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const sharedSeq = new SharedObjectSequence(runtime, id, attributes); - await sharedSeq.loadLocal(objectStorage); - return sharedSeq; - } - public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedString = new SharedObjectSequence(document, id, this.attributes); sharedString.initializeLocal(); @@ -170,17 +147,6 @@ export class SharedNumberSequenceFactory implements IChannelFactory { return sharedSeq; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const sharedSeq = new SharedNumberSequence(runtime, id, attributes); - await sharedSeq.loadLocal(objectStorage); - return sharedSeq; - } - public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedString = new SharedNumberSequence(document, id, this.attributes); sharedString.initializeLocal(); diff --git a/packages/dds/sequence/src/sharedIntervalCollection.ts b/packages/dds/sequence/src/sharedIntervalCollection.ts index cea03ae2d9ef..b79ebf871105 100644 --- a/packages/dds/sequence/src/sharedIntervalCollection.ts +++ b/packages/dds/sequence/src/sharedIntervalCollection.ts @@ -63,18 +63,6 @@ export class SharedIntervalCollectionFactory implements IChannelFactory { return map; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const map = new SharedIntervalCollection(id, runtime, attributes); - await map.loadLocal(objectStorage); - - return map; - } - public create(runtime: IFluidDataStoreRuntime, id: string): SharedIntervalCollection { const map = new SharedIntervalCollection( id, diff --git a/packages/dds/sequence/src/sparsematrix.ts b/packages/dds/sequence/src/sparsematrix.ts index 5756074d3424..218f160ed9e1 100644 --- a/packages/dds/sequence/src/sparsematrix.ts +++ b/packages/dds/sequence/src/sparsematrix.ts @@ -19,7 +19,6 @@ import { Jsonable, JsonablePrimitive, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ISharedObject } from "@fluidframework/shared-object-base"; import { pkgVersion } from "./packageVersion"; @@ -365,17 +364,6 @@ export class SparseMatrixFactory implements IChannelFactory { return sharedObject; } - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const sharedObject = new SparseMatrix(runtime, id, attributes); - await sharedObject.loadLocal(objectStorage); - return sharedObject; - } - public create(document: IFluidDataStoreRuntime, id: string): ISharedObject { const sharedObject = new SparseMatrix(document, id, this.attributes); sharedObject.initializeLocal(); diff --git a/packages/dds/shared-object-base/src/sharedObject.ts b/packages/dds/shared-object-base/src/sharedObject.ts index 58c80853d2b8..695051a5210b 100644 --- a/packages/dds/shared-object-base/src/sharedObject.ts +++ b/packages/dds/shared-object-base/src/sharedObject.ts @@ -130,26 +130,18 @@ export abstract class SharedObject { - this.services = services; - + branchId: string | undefined, + services: IChannelServices, + ): Promise { + if (this.runtime.attachState !== AttachState.Detached) { + this.services = services; + } await this.loadCore( branchId, services.objectStorage); - this.attachDeltaHandler(); - } - - /** - * Loads the given channel. This is similar to load however it is used to load a local channel from a snapshot. - * @param storage - Storage service to read objects at a given path. This is not connected to storage - * endpoint but have blobs to read from. - */ - public async loadLocal( - objectStorage: IChannelStorageService): Promise { - await this.loadCore( - undefined, - objectStorage); + if (this.runtime.attachState !== AttachState.Detached) { + this.attachDeltaHandler(); + } } /** diff --git a/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts b/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts index e8aab3f3e726..9d02c6f4a719 100644 --- a/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts +++ b/packages/dds/shared-summary-block/src/sharedSummaryBlockFactory.ts @@ -8,7 +8,6 @@ import { IFluidDataStoreRuntime, IChannelServices, IChannelFactory, - IChannelStorageService, } from "@fluidframework/datastore-definitions"; import { ISharedObject, @@ -64,21 +63,6 @@ export class SharedSummaryBlockFactory implements IChannelFactory { return sharedSummaryBlock; } - /** - * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal} - */ - public async loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - objectStorage: IChannelStorageService, - attributes: IChannelAttributes, - ): Promise { - const sharedSummaryBlock = new SharedSummaryBlock(id, runtime, attributes); - await sharedSummaryBlock.loadLocal(objectStorage); - - return sharedSummaryBlock; - } - /** * {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create} */ diff --git a/packages/runtime/component-runtime-definitions/src/channel.ts b/packages/runtime/component-runtime-definitions/src/channel.ts new file mode 100644 index 000000000000..60a1cac6c34e --- /dev/null +++ b/packages/runtime/component-runtime-definitions/src/channel.ts @@ -0,0 +1,187 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +import { IFluidLoadable } from "@fluidframework/component-core-interfaces"; +import { ISequencedDocumentMessage, ITree } from "@fluidframework/protocol-definitions"; +import { IChannelAttributes } from "./storage"; +import { IFluidDataStoreRuntime } from "./componentRuntime"; + +declare module "@fluidframework/component-core-interfaces" { + // eslint-disable-next-line @typescript-eslint/no-empty-interface + interface IComponent extends Readonly> { } + + // eslint-disable-next-line @typescript-eslint/no-empty-interface + interface IFluidObject extends Readonly> { } +} + +export const IChannel: keyof IProvideChannel = "IChannel"; + +export interface IProvideChannel { + readonly IChannel: IChannel; +} + +export interface IChannel extends IProvideChannel, IFluidLoadable { + /** + * A readonly identifier for the channel + */ + readonly id: string; + + readonly owner?: string; + + readonly attributes: IChannelAttributes; + + /** + * Generates snapshot of the channel. + */ + snapshot(): ITree; + + /** + * True if the data structure is attached to storage. + */ + isAttached(): boolean; + + /** + * Enables the channel to send and receive ops + */ + connect(services: IChannelServices): void; +} + +/** + * Handler provided by shared data structure to process requests from the runtime. + */ +export interface IDeltaHandler { + /** + * Processes the op. + * @param message - The message to process + * @param local - Whether the message originated from the local client + * @param localOpMetadata - For local client messages, this is the metadata that was submitted with the message. + * For messages from a remote client, this will be undefined. + */ + process: (message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown) => void; + + /** + * State change events to indicate changes to the delta connection + * @param connected - true if connected, false otherwise + */ + setConnectionState(connected: boolean): void; + + /** + * Called when the runtime asks the client to resubmit an op. This may be because the Container reconnected and + * this op was not acked. + * The client can choose to resubmit the same message, submit different / multiple messages or not submit anything + * at all. + * @param message - The original message that was submitted. + * @param localOpMetadata - The local metadata associated with the original message. + */ + reSubmit(message: any, localOpMetadata: unknown): void; +} + +/** + * Interface to represent a connection to a delta notification stream. + */ +export interface IDeltaConnection { + connected: boolean; + + /** + * Send new messages to the server. + * @param messageContent - The content of the message to be sent. + * @param localOpMetadata - The local metadata associated with the message. This is kept locally by the runtime + * and not sent to the server. It will be provided back when this message is acknowledged by the server. It will + * also be provided back when asked to resubmit the message. + */ + submit(messageContent: any, localOpMetadata: unknown): void; + + /** + * Attaches a message handler to the delta connection + */ + attach(handler: IDeltaHandler): void; + + /** + * Indicates that the channel is dirty and needs to be part of the summary. It is called by a SharedSummaryBlock + * that needs to be part of the summary but does not generate ops. + */ + dirty(): void; +} + +/** + * Storage services to read the objects at a given path. + */ +export interface IChannelStorageService { + /** + * Reads the object contained at the given path. Returns a base64 string representation for the object. + */ + read(path: string): Promise; + + /** + * Determines if there is an object contained at the given path. + */ + contains(path: string): Promise; + + /** + * Lists the blobs that exist at a specific path. + */ + list(path: string): Promise; +} + +/** + * Storage services to read the objects at a given path using the given delta connection. + */ +export interface IChannelServices { + deltaConnection: IDeltaConnection; + + objectStorage: IChannelStorageService; +} + +/** + * Definitions of a channel factory. Factories follow a common model but enable custom behavior. + */ +export interface IChannelFactory { + /** + * String representing the type of the factory. + */ + readonly type: string; + + /** + * Attributes of the channel. + */ + readonly attributes: IChannelAttributes; + + /** + * Loads the given channel. This call is only ever invoked internally as the only thing + * that is ever directly loaded is the document itself. Load will then only be called on documents that + * were created and added to a channel. + * @param runtime - Component runtime containing state/info/helper methods about the component. + * @param id - ID of the channel. + * @param services - Services to read objects at a given path using the delta connection. + * @param branchId - The branch ID. + * @param channelAttributes - The attributes for the the channel to be loaded. + * @returns The loaded object + * + * @privateRemarks + * Thought: should the storage object include the version information and limit access to just files + * for the given object? The latter seems good in general. But both are probably good things. We then just + * need a way to allow the document to provide later storage for the object. + */ + load( + runtime: IFluidDataStoreRuntime, + id: string, + services: IChannelServices, + branchId: string, + channelAttributes: Readonly, + ): Promise; + + /** + * Creates a local version of the channel. + * Calling attach on the object later will insert it into the object stream. + * @param runtime - The runtime the new object will be associated with + * @param id - The unique ID of the new object + * @returns The newly created object. + * + * @privateRemarks + * NOTE here - When we attach we need to submit all the pending ops prior to actually doing the attach + * for consistency. + */ + create(runtime: IFluidDataStoreRuntime, id: string): IChannel; +} diff --git a/packages/runtime/datastore-definitions/src/channel.ts b/packages/runtime/datastore-definitions/src/channel.ts index 322c784b608b..1243d47b7e5d 100644 --- a/packages/runtime/datastore-definitions/src/channel.ts +++ b/packages/runtime/datastore-definitions/src/channel.ts @@ -165,7 +165,7 @@ export interface IChannelFactory { runtime: IFluidDataStoreRuntime, id: string, services: IChannelServices, - branchId: string, + branchId: string | undefined, channelAttributes: Readonly, ): Promise; @@ -181,21 +181,4 @@ export interface IChannelFactory { * for consistency. */ create(runtime: IFluidDataStoreRuntime, id: string): IChannel; - - /** - * Loads the given channel. This is similar to load however it is used to load a local channel from a snapshot. - * This call is only ever invoked internally as the only thing that is ever directly loaded is the document itself. - * @param runtime - Component runtime containing state/info/helper methods about the component. - * @param id - ID of the channel. - * @param storage - Storage service to read objects at a given path. This is not connected to storage - * endpoint but have blobs to read from. - * @param channelAttributes - The attributes for the the channel to be loaded. - * @returns The loaded object - */ - loadLocal( - runtime: IFluidDataStoreRuntime, - id: string, - storage: IChannelStorageService, - channelAttributes: Readonly, - ): Promise; } diff --git a/packages/runtime/datastore/src/localChannelContext.ts b/packages/runtime/datastore/src/localChannelContext.ts index 31bbb414536f..0c44e1b87a83 100644 --- a/packages/runtime/datastore/src/localChannelContext.ts +++ b/packages/runtime/datastore/src/localChannelContext.ts @@ -122,10 +122,11 @@ export class LocalChannelContext implements IChannelContext { ".attributes"); assert(this.factory, "Factory should be there for local channel"); - const channel = await this.factory.loadLocal( + const channel = await this.factory.load( this.runtime, this.id, - this.services.objectStorage, + this.services, + undefined, attributes); // Commit changes. From 59a4ff2a9dfb8bb8368d799d765a3e50bc7018e1 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 5 Aug 2020 01:08:29 -0700 Subject: [PATCH 10/27] pr sugg --- packages/loader/container-definitions/src/loader.ts | 4 ++-- packages/loader/container-definitions/src/runtime.ts | 2 +- packages/loader/container-loader/src/container.ts | 8 ++++---- .../loader/container-loader/src/containerContext.ts | 4 ++-- packages/loader/container-loader/src/loader.ts | 6 +++--- .../execution-context-loader/src/webWorkerLoader.ts | 4 ++-- .../container-runtime/src/containerRuntime.ts | 8 +++++--- .../src/test/attachRegisterLocalApiTests.spec.ts | 4 ++-- .../src/test/deRehydrateContainerTests.spec.ts | 12 ++++++------ .../src/test/detachedContainerTests.spec.ts | 4 ++-- .../test/service-load-test/src/nodeStressTest.ts | 4 ++-- .../src/getTinyliciousContainer.ts | 4 ++-- .../tools/webpack-component-loader/src/loader.ts | 4 ++-- 13 files changed, 35 insertions(+), 33 deletions(-) diff --git a/packages/loader/container-definitions/src/loader.ts b/packages/loader/container-definitions/src/loader.ts index 844a11a9f1d0..26ad0775095f 100644 --- a/packages/loader/container-definitions/src/loader.ts +++ b/packages/loader/container-definitions/src/loader.ts @@ -72,7 +72,7 @@ export interface ICodeAllowList { /** * Source to create the detached container from. Either needs the codeDetails or a snapshot to start from. */ -export type IDetachedContainerSource = { +export type DetachedContainerSource = { codeDetails: IFluidCodeDetails, useSnapshot: false, } | { @@ -159,7 +159,7 @@ export interface ILoader { * Creates a new container using the specified chaincode but in an unattached state. While unattached all * updates will only be local until the user explicitly attaches the container to a service provider. */ - createDetachedContainer(source: IDetachedContainerSource): Promise; + createDetachedContainer(source: DetachedContainerSource): Promise; } export enum LoaderHeader { diff --git a/packages/loader/container-definitions/src/runtime.ts b/packages/loader/container-definitions/src/runtime.ts index 2253f24d19cc..cc2e9651bfb4 100644 --- a/packages/loader/container-definitions/src/runtime.ts +++ b/packages/loader/container-definitions/src/runtime.ts @@ -111,7 +111,7 @@ export interface IContainerContext extends IMessageScheduler, IDisposable { readonly storage: IDocumentStorageService | undefined | null; readonly connected: boolean; readonly branch: string; - readonly baseSnapshot: ISnapshotTree | null; + readonly baseSnapshot: ISnapshotTree | undefined; readonly submitFn: (type: MessageType, contents: any, batch: boolean, appData?: any) => number; readonly submitSignalFn: (contents: any) => void; readonly snapshotFn: (message: string) => Promise; diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 894cf51cc9fa..46c8ceafcbfc 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -29,7 +29,7 @@ import { ContainerWarning, IThrottlingWarning, AttachState, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { performanceNow } from "@fluidframework/common-utils"; import { @@ -195,7 +195,7 @@ export class Container extends EventEmitterWithErrorHandling i options: any, scope: IFluidObject & IFluidObject, loader: Loader, - source: IDetachedContainerSource, + source: DetachedContainerSource, serviceFactory: IDocumentServiceFactory, urlResolver: IUrlResolver, logger?: ITelemetryBaseLogger, @@ -1494,7 +1494,7 @@ export class Container extends EventEmitterWithErrorHandling i this.scope, this.codeLoader, chaincode, - snapshot ?? null, + snapshot, attributes, this.blobManager, new DeltaManagerProxy(this._deltaManager), @@ -1532,7 +1532,7 @@ export class Container extends EventEmitterWithErrorHandling i this.scope, this.codeLoader, runtimeFactory, - snapshot ?? null, + snapshot, attributes, this.blobManager, new DeltaManagerProxy(this._deltaManager), diff --git a/packages/loader/container-loader/src/containerContext.ts b/packages/loader/container-loader/src/containerContext.ts index 2ca4369d1f01..90c04f76107c 100644 --- a/packages/loader/container-loader/src/containerContext.ts +++ b/packages/loader/container-loader/src/containerContext.ts @@ -50,7 +50,7 @@ export class ContainerContext implements IContainerContext { scope: IFluidObject & IFluidObject, codeLoader: ICodeLoader, runtimeFactory: IRuntimeFactory, - baseSnapshot: ISnapshotTree | null, + baseSnapshot: ISnapshotTree | undefined, attributes: IDocumentAttributes, blobManager: BlobManager | undefined, deltaManager: IDeltaManager, @@ -170,7 +170,7 @@ export class ContainerContext implements IContainerContext { public readonly scope: IFluidObject & IFluidObject, public readonly codeLoader: ICodeLoader, public readonly runtimeFactory: IRuntimeFactory, - private readonly _baseSnapshot: ISnapshotTree | null, + private readonly _baseSnapshot: ISnapshotTree | undefined, private readonly attributes: IDocumentAttributes, public readonly blobManager: BlobManager | undefined, public readonly deltaManager: IDeltaManager, diff --git a/packages/loader/container-loader/src/loader.ts b/packages/loader/container-loader/src/loader.ts index c8aff1b1cd96..e441bda68f1f 100644 --- a/packages/loader/container-loader/src/loader.ts +++ b/packages/loader/container-loader/src/loader.ts @@ -17,7 +17,7 @@ import { ILoader, IProxyLoaderFactory, LoaderHeader, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Deferred, performanceNow } from "@fluidframework/common-utils"; import { ChildLogger, DebugLogger, PerformanceEvent } from "@fluidframework/telemetry-utils"; @@ -99,7 +99,7 @@ export class RelativeLoader extends EventEmitter implements ILoader { return this.loader.request(request); } - public async createDetachedContainer(source: IDetachedContainerSource): Promise { + public async createDetachedContainer(source: DetachedContainerSource): Promise { throw new Error("Relative loader should not create a detached container"); } @@ -156,7 +156,7 @@ export class Loader extends EventEmitter implements ILoader { this.documentServiceFactory = MultiDocumentServiceFactory.create(documentServiceFactory); } - public async createDetachedContainer(source: IDetachedContainerSource): Promise { + public async createDetachedContainer(source: DetachedContainerSource): Promise { debug(`Container creating in detached state: ${performanceNow()} `); return Container.create( diff --git a/packages/loader/execution-context-loader/src/webWorkerLoader.ts b/packages/loader/execution-context-loader/src/webWorkerLoader.ts index c18a0e59d69f..5820ba4384c3 100644 --- a/packages/loader/execution-context-loader/src/webWorkerLoader.ts +++ b/packages/loader/execution-context-loader/src/webWorkerLoader.ts @@ -9,7 +9,7 @@ import { IRequest, IResponse, } from "@fluidframework/component-core-interfaces"; -import { IContainer, ILoader, IDetachedContainerSource } from "@fluidframework/container-definitions"; +import { IContainer, ILoader, DetachedContainerSource } from "@fluidframework/container-definitions"; import { IFluidResolvedUrl } from "@fluidframework/driver-definitions"; import Comlink from "comlink"; @@ -71,7 +71,7 @@ export class WebWorkerLoader implements ILoader, IFluidRunnable, IFluidRouter { return this.proxy.resolve(request); } - public async createDetachedContainer(source: IDetachedContainerSource): Promise { + public async createDetachedContainer(source: DetachedContainerSource): Promise { return this.proxy.createDetachedContainer(source); } } diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index d0d14af50c02..6442b5d47d70 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -493,7 +493,7 @@ export class ContainerRuntime extends EventEmitter // Create all internal components if not already existing on storage or loaded a detached // container from snapshot(ex. draft mode). - if (!(context.existing || context.baseSnapshot !== null)) { + if (!(context.existing || context.baseSnapshot)) { await runtime.createRootDataStore(schedulerId, schedulerId); } @@ -738,7 +738,8 @@ export class ContainerRuntime extends EventEmitter // Extract components stored inside the snapshot const components = new Map(); - if (context.baseSnapshot) { + // back-compat 0.24 baseSnapshotCouldBeNull + if (context.baseSnapshot !== undefined && context.baseSnapshot !== null) { const baseSnapshot = context.baseSnapshot; Object.keys(baseSnapshot.trees).forEach((value) => { if (value !== ".protocol" && value !== ".logTail" && value !== ".serviceProtocol") { @@ -754,7 +755,8 @@ export class ContainerRuntime extends EventEmitter // If it is loaded from a snapshot but in detached state, then create a local component. if (this.attachState === AttachState.Detached) { let pkgFromSnapshot: string[]; - if (context.baseSnapshot === null) { + // back-compat 0.24 baseSnapshotCouldBeNull + if (context.baseSnapshot === null || context.baseSnapshot === undefined) { throw new Error("Snapshot should be there to load from!!"); } const snapshotTree = value as ISnapshotTree; diff --git a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts index 35f419d9478a..4191fe98eb91 100644 --- a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts @@ -9,7 +9,7 @@ import { IFluidCodeDetails, IProxyLoaderFactory, AttachState, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -32,7 +32,7 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const source: IDetachedContainerSource = { + const source: DetachedContainerSource = { codeDetails, useSnapshot: false, }; diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index c6e3665a31cd..f5955743c31e 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -7,7 +7,7 @@ import assert from "assert"; import { IFluidCodeDetails, IProxyLoaderFactory, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -38,7 +38,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const source: IDetachedContainerSource = { + const source: DetachedContainerSource = { codeDetails, useSnapshot: false, }; @@ -195,7 +195,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { await createDetachedContainerAndGetRootComponent(); const snapshotTree: string = container.serialize(); - const rehydrationSource: IDetachedContainerSource = { + const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, }; @@ -242,7 +242,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { await createDetachedContainerAndGetRootComponent(); const snapshotTree: string = container.serialize(); - const rehydrationSource: IDetachedContainerSource = { + const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, }; @@ -295,7 +295,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { sharedStringBefore.insertText(0, "Hello"); const snapshotTree: string = container.serialize(); - const rehydrationSource: IDetachedContainerSource = { + const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, }; @@ -317,7 +317,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); sharedString1.insertText(0, str); const snapshotTree: string = container.serialize(); - const rehydrationSource: IDetachedContainerSource = { + const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, }; diff --git a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts index be1731f38b8f..7c88f09abd57 100644 --- a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -9,7 +9,7 @@ import { IFluidCodeDetails, IProxyLoaderFactory, AttachState, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { ConnectionState, Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -41,7 +41,7 @@ describe("Detached Container", () => { package: "detachedContainerTestPackage", config: {}, }; - const source: IDetachedContainerSource = { + const source: DetachedContainerSource = { codeDetails: pkg, useSnapshot: false, }; diff --git a/packages/test/service-load-test/src/nodeStressTest.ts b/packages/test/service-load-test/src/nodeStressTest.ts index 4bd4182ee9f9..8f3b20beaa45 100644 --- a/packages/test/service-load-test/src/nodeStressTest.ts +++ b/packages/test/service-load-test/src/nodeStressTest.ts @@ -9,7 +9,7 @@ import commander from "commander"; import { IProxyLoaderFactory, IFluidCodeDetails, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { OdspDocumentServiceFactory, OdspDriverUrlResolver } from "@fluidframework/odsp-driver"; @@ -42,7 +42,7 @@ const codeDetails: IFluidCodeDetails = { config: {}, }; -const source: IDetachedContainerSource = { +const source: DetachedContainerSource = { codeDetails, useSnapshot: false, }; diff --git a/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts b/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts index a3d32a1ed107..e31f9d5bb105 100644 --- a/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts +++ b/packages/tools/get-tinylicious-container/src/getTinyliciousContainer.ts @@ -5,7 +5,7 @@ import { IRequest } from "@fluidframework/component-core-interfaces"; import { - IRuntimeFactory, IDetachedContainerSource, + IRuntimeFactory, DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Container, Loader } from "@fluidframework/container-loader"; import { @@ -100,7 +100,7 @@ export async function getTinyliciousContainer( // We're not actually using the code proposal (our code loader always loads the same module regardless of the // proposal), but the Container will only give us a NullRuntime if there's no proposal. So we'll use a fake // proposal. - const source: IDetachedContainerSource = { + const source: DetachedContainerSource = { codeDetails: { package: "", config: {} }, useSnapshot: false, }; diff --git a/packages/tools/webpack-component-loader/src/loader.ts b/packages/tools/webpack-component-loader/src/loader.ts index dc79d3d5d2be..d8ccf61924b1 100644 --- a/packages/tools/webpack-component-loader/src/loader.ts +++ b/packages/tools/webpack-component-loader/src/loader.ts @@ -12,7 +12,7 @@ import { IFluidCodeResolver, IResolvedFluidCodeDetails, isFluidPackage, - IDetachedContainerSource, + DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Container } from "@fluidframework/container-loader"; import { IDocumentServiceFactory } from "@fluidframework/driver-definitions"; @@ -169,7 +169,7 @@ export async function start( if (!codeDetails) { throw new Error("Code details must be defined for detached mode!!"); } - const source: IDetachedContainerSource = { + const source: DetachedContainerSource = { codeDetails, useSnapshot: false, }; From 737111dfe17a90f6ed4d345cd229651a0b2a79ad Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 5 Aug 2020 01:23:12 -0700 Subject: [PATCH 11/27] pr sugg --- packages/dds/merge-tree/src/snapshotLoader.ts | 33 ++++++++++--------- .../runtime/agent-scheduler/src/scheduler.ts | 2 +- .../src/dataStoreRuntime.ts | 3 +- .../runtime/datastore/src/dataStoreRuntime.ts | 5 ++- .../runtime/test-runtime-utils/src/mocks.ts | 4 +-- .../test/test-utils/src/testFluidComponent.ts | 2 +- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/dds/merge-tree/src/snapshotLoader.ts b/packages/dds/merge-tree/src/snapshotLoader.ts index db2631f5cfd4..9d30d58de6c7 100644 --- a/packages/dds/merge-tree/src/snapshotLoader.ts +++ b/packages/dds/merge-tree/src/snapshotLoader.ts @@ -135,23 +135,26 @@ export class SnapshotLoader { if (chunk.headerMetadata === undefined) { throw new Error("header metadata not available"); } - // specify a default client id, "snapshot" here as we - // should enter collaboration/op sending mode if we load - // a snapshot in any case (summary or attach message) - // once we get a client id this will be called with that - // clientId in the connected event - // However if we load a detached container from snapshot, then we don't supply a default clientId + // If we load a detached container from snapshot, then we don't supply a default clientId // because we don't want to start collaboration. - this.client.startOrUpdateCollaboration( - this.runtime.attachState === AttachState.Detached ? undefined : this.runtime.clientId ?? "snapshot", + if (this.runtime.attachState !== AttachState.Detached) { + // specify a default client id, "snapshot" here as we + // should enter collaboration/op sending mode if we load + // a snapshot in any case (summary or attach message) + // once we get a client id this will be called with that + // clientId in the connected event + this.client.startOrUpdateCollaboration( + this.runtime.clientId ?? "snapshot", + + // TODO: Make 'minSeq' non-optional once the new snapshot format becomes the default? + // (See https://github.com/microsoft/FluidFramework/issues/84) + /* minSeq: */ chunk.headerMetadata.minSequenceNumber !== undefined + ? chunk.headerMetadata.minSequenceNumber + : chunk.headerMetadata.sequenceNumber, + /* currentSeq: */ chunk.headerMetadata.sequenceNumber, + branching); + } - // TODO: Make 'minSeq' non-optional once the new snapshot format becomes the default? - // (See https://github.com/microsoft/FluidFramework/issues/84) - /* minSeq: */ chunk.headerMetadata.minSequenceNumber !== undefined - ? chunk.headerMetadata.minSequenceNumber - : chunk.headerMetadata.sequenceNumber, - /* currentSeq: */ chunk.headerMetadata.sequenceNumber, - branching); return chunk; } diff --git a/packages/runtime/agent-scheduler/src/scheduler.ts b/packages/runtime/agent-scheduler/src/scheduler.ts index 1dce7f34c563..19330ef22a28 100644 --- a/packages/runtime/agent-scheduler/src/scheduler.ts +++ b/packages/runtime/agent-scheduler/src/scheduler.ts @@ -37,7 +37,7 @@ class AgentScheduler extends EventEmitter implements IAgentScheduler, IFluidRout let root: ISharedMap; let scheduler: ConsensusRegisterCollection; // Don't create if existing on storage or loaded detached from snapshot(ex. draft mode). - if (!(runtime.existing || runtime.baseSnapshot)) { + if (!(runtime.existing || runtime.isLoadedFromSnapshot)) { root = SharedMap.create(runtime, "root"); root.bindToContext(); scheduler = ConsensusRegisterCollection.create(runtime); diff --git a/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts b/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts index 68c8596554d4..2ee1060be3bf 100644 --- a/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts @@ -22,7 +22,6 @@ import { IDocumentMessage, IQuorum, ISequencedDocumentMessage, - ISnapshotTree, } from "@fluidframework/protocol-definitions"; import { IInboundSignalMessage, IProvideFluidDataStoreRegistry } from "@fluidframework/runtime-definitions"; import { IChannel } from "."; @@ -52,7 +51,7 @@ export interface IFluidDataStoreRuntime extends readonly existing: boolean; - readonly baseSnapshot: ISnapshotTree | undefined; + readonly isLoadedFromSnapshot: boolean; readonly parentBranch: string | null; diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index ce847d046868..54aef095e043 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -38,7 +38,6 @@ import { IQuorum, ISequencedDocumentMessage, ITreeEntry, - ISnapshotTree, ITree, } from "@fluidframework/protocol-definitions"; import { @@ -145,8 +144,8 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto return this._attachState; } - public get baseSnapshot(): ISnapshotTree | undefined { - return this.dataStoreContext.baseSnapshot; + public get isLoadedFromSnapshot(): boolean { + return this.dataStoreContext.baseSnapshot !== undefined; } /** diff --git a/packages/runtime/test-runtime-utils/src/mocks.ts b/packages/runtime/test-runtime-utils/src/mocks.ts index 3c8307d27302..1b071d8a1898 100644 --- a/packages/runtime/test-runtime-utils/src/mocks.ts +++ b/packages/runtime/test-runtime-utils/src/mocks.ts @@ -395,8 +395,8 @@ export class MockFluidDataStoreRuntime extends EventEmitter this._local = local; } - public get baseSnapshot() { - return undefined; + public get isLoadedFromSnapshot() { + return false; } private _disposed = false; diff --git a/packages/test/test-utils/src/testFluidComponent.ts b/packages/test/test-utils/src/testFluidComponent.ts index cb16c164cd02..f8c8645bed5e 100644 --- a/packages/test/test-utils/src/testFluidComponent.ts +++ b/packages/test/test-utils/src/testFluidComponent.ts @@ -91,7 +91,7 @@ export class TestFluidComponent implements ITestFluidComponent { private async initialize() { // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). - if (!(this.runtime.existing || this.runtime.baseSnapshot !== undefined)) { + if (!(this.runtime.existing || this.runtime.isLoadedFromSnapshot)) { this.root = SharedMap.create(this.runtime, "root"); this.factoryEntriesMap.forEach((sharedObjectFactory: IChannelFactory, key: string) => { From cfde78ccec5b36fb68320a11429de9f8c332094e Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 5 Aug 2020 01:27:47 -0700 Subject: [PATCH 12/27] pr sugg --- packages/dds/merge-tree/src/snapshotLoader.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/dds/merge-tree/src/snapshotLoader.ts b/packages/dds/merge-tree/src/snapshotLoader.ts index 9d30d58de6c7..46497e0c84cd 100644 --- a/packages/dds/merge-tree/src/snapshotLoader.ts +++ b/packages/dds/merge-tree/src/snapshotLoader.ts @@ -155,7 +155,6 @@ export class SnapshotLoader { branching); } - return chunk; } From acc535adc60fdc320d5ac7c20e51319eb24f95f2 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 5 Aug 2020 10:44:54 -0700 Subject: [PATCH 13/27] fix name --- .../end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index f5955743c31e..3d6db15c25cd 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -23,7 +23,7 @@ import { SharedMap, SharedDirectory } from "@fluidframework/map"; import { IDocumentAttributes } from "@fluidframework/protocol-definitions"; import { IContainerRuntime } from "@fluidframework/container-runtime-definitions"; import { IContainerRuntimeBase } from "@fluidframework/runtime-definitions"; -import { IRequest } from "@fluidframework/component-core-interfaces"; +import { IRequest } from "@fluidframework/core-interfaces"; import { SharedString, SparseMatrix } from "@fluidframework/sequence"; import { ConsensusRegisterCollection } from "@fluidframework/register-collection"; import { ConsensusQueue, ConsensusOrderedCollection } from "@fluidframework/ordered-collection"; From 84bc42e8d65cf32ff3bac3f6061dfc5b0b0e2024 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Thu, 6 Aug 2020 13:18:45 -0700 Subject: [PATCH 14/27] pr sugg --- packages/loader/container-definitions/src/loader.ts | 3 ++- packages/loader/container-loader/src/container.ts | 7 +++++-- .../runtime/container-runtime/src/containerRuntime.ts | 4 ++-- .../src/test/deRehydrateContainerTests.spec.ts | 8 ++++---- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/loader/container-definitions/src/loader.ts b/packages/loader/container-definitions/src/loader.ts index 1e9e0540d0e6..a4dea3a1e2ca 100644 --- a/packages/loader/container-definitions/src/loader.ts +++ b/packages/loader/container-definitions/src/loader.ts @@ -9,6 +9,7 @@ import { IDocumentMessage, IQuorum, ISequencedDocumentMessage, + ISnapshotTree, } from "@fluidframework/protocol-definitions"; import { IResolvedUrl } from "@fluidframework/driver-definitions"; import { IEvent, IEventProvider } from "@fluidframework/common-definitions"; @@ -76,7 +77,7 @@ export type DetachedContainerSource = { codeDetails: IFluidCodeDetails, useSnapshot: false, } | { - snapshot: string; + snapshot: ISnapshotTree; useSnapshot: true; }; diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 73b9518f0ed8..5cdd72f13e7b 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -209,9 +209,12 @@ export class Container extends EventEmitterWithErrorHandling i urlResolver, {}, logger); - // eslint-disable-next-line @typescript-eslint/no-unused-expressions - source.useSnapshot ? await container.createDetachedFromSnapshot(JSON.parse(source.snapshot)) : + + if (source.useSnapshot) { + await container.createDetachedFromSnapshot(source.snapshot); + } else { await container.createDetached(source.codeDetails); + } return container; } diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 727d4ffc9d9c..611fb6b76380 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -102,7 +102,7 @@ import { LocalFluidDataStoreContext, RemotedFluidDataStoreContext, currentSnapshotFormatVersion, - IFluidDataStoretAttributes, + IFluidDataStoreAttributes, } from "./dataStoreContext"; import { FluidHandleContext } from "./dataStoreHandleContext"; import { FluidDataStoreRegistry } from "./dataStoreRegistry"; @@ -762,7 +762,7 @@ export class ContainerRuntime extends EventEmitter } const snapshotTree = value as ISnapshotTree; // Need to rip through snapshot. - const { pkg, snapshotFormatVersion } = readAndParseFromBlobs( + const { pkg, snapshotFormatVersion } = readAndParseFromBlobs( snapshotTree.blobs, snapshotTree.blobs[".component"]); // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 3d6db15c25cd..24d33a27edb9 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -194,7 +194,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const { container } = await createDetachedContainerAndGetRootComponent(); - const snapshotTree: string = container.serialize(); + const snapshotTree = JSON.parse(container.serialize()); const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, @@ -241,7 +241,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const { container } = await createDetachedContainerAndGetRootComponent(); - const snapshotTree: string = container.serialize(); + const snapshotTree = JSON.parse(container.serialize()); const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, @@ -294,7 +294,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); sharedStringBefore.insertText(0, "Hello"); - const snapshotTree: string = container.serialize(); + const snapshotTree = JSON.parse(container.serialize()); const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, @@ -316,7 +316,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const defaultComponent1 = response1.value as ITestFluidComponent; const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); sharedString1.insertText(0, str); - const snapshotTree: string = container.serialize(); + const snapshotTree = JSON.parse(container.serialize()); const rehydrationSource: DetachedContainerSource = { snapshot: snapshotTree, useSnapshot: true, From 7150af4d52c10d3b7b51c1daf8b99ddce1d8ddde Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 12 Aug 2020 12:46:14 -0700 Subject: [PATCH 15/27] separate the 2 apis --- .../src/getTinyliciousContainer.ts | 11 ++---- .../container-definitions/src/loader.ts | 19 ++++------ .../loader/container-loader/src/container.ts | 12 +++--- .../loader/container-loader/src/loader.ts | 28 ++++++++++++-- .../execution-context-loader/package.json | 1 + .../src/webWorkerLoader.ts | 9 ++++- .../test/attachRegisterLocalApiTests.spec.ts | 8 +--- .../test/deRehydrateContainerTests.spec.ts | 33 ++++------------ .../src/test/detachedContainerTests.spec.ts | 38 +++++++++---------- .../service-load-test/src/nodeStressTest.ts | 8 +--- .../webpack-component-loader/src/loader.ts | 10 +---- 11 files changed, 78 insertions(+), 99 deletions(-) diff --git a/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts b/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts index 7473582ffebb..d9ea6833f19e 100644 --- a/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts +++ b/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts @@ -4,9 +4,7 @@ */ import { IRequest } from "@fluidframework/core-interfaces"; -import { - IRuntimeFactory, DetachedContainerSource, -} from "@fluidframework/container-definitions"; +import { IRuntimeFactory } from "@fluidframework/container-definitions"; import { Container, Loader } from "@fluidframework/container-loader"; import { IFluidResolvedUrl, @@ -100,11 +98,8 @@ export async function getTinyliciousContainer( // We're not actually using the code proposal (our code loader always loads the same module regardless of the // proposal), but the Container will only give us a NullRuntime if there's no proposal. So we'll use a fake // proposal. - const source: DetachedContainerSource = { - codeDetails: { package: "", config: {} }, - useSnapshot: false, - }; - container = await loader.createDetachedContainer(source); + const codeDetails = { package: "", config: {} }; + container = await loader.createDetachedContainer(codeDetails); await container.attach({ url: documentId }); } else { // The InsecureTinyliciousUrlResolver expects the url of the request to be the documentId. diff --git a/packages/loader/container-definitions/src/loader.ts b/packages/loader/container-definitions/src/loader.ts index a4dea3a1e2ca..ef8903c753ad 100644 --- a/packages/loader/container-definitions/src/loader.ts +++ b/packages/loader/container-definitions/src/loader.ts @@ -70,17 +70,6 @@ export interface ICodeAllowList { testSource(source: IResolvedFluidCodeDetails): Promise; } -/** - * Source to create the detached container from. Either needs the codeDetails or a snapshot to start from. - */ -export type DetachedContainerSource = { - codeDetails: IFluidCodeDetails, - useSnapshot: false, -} | { - snapshot: ISnapshotTree; - useSnapshot: true; -}; - export interface IContainerEvents extends IEvent { (event: "readonly", listener: (readonly: boolean) => void): void; (event: "connected", listener: (clientId: string) => void); @@ -160,7 +149,13 @@ export interface ILoader { * Creates a new container using the specified chaincode but in an unattached state. While unattached all * updates will only be local until the user explicitly attaches the container to a service provider. */ - createDetachedContainer(source: DetachedContainerSource): Promise; + createDetachedContainer(source: IFluidCodeDetails): Promise; + + /** + * Creates a new container using the specified snapshot but in an unattached state. While unattached all + * updates will only be local until the user explicitly attaches the container to a service provider. + */ + createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise; } export enum LoaderHeader { diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index fc879b539b75..a4ec53afbe68 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -29,7 +29,6 @@ import { ContainerWarning, IThrottlingWarning, AttachState, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { performanceNow } from "@fluidframework/common-utils"; import { @@ -195,7 +194,8 @@ export class Container extends EventEmitterWithErrorHandling i options: any, scope: IFluidObject, loader: Loader, - source: DetachedContainerSource, + codeDetails: IFluidCodeDetails | undefined, + snapshot: ISnapshotTree | undefined, serviceFactory: IDocumentServiceFactory, urlResolver: IUrlResolver, logger?: ITelemetryBaseLogger, @@ -210,11 +210,13 @@ export class Container extends EventEmitterWithErrorHandling i {}, logger); - if (source.useSnapshot) { - await container.createDetachedFromSnapshot(source.snapshot); + if (snapshot !== undefined) { + await container.createDetachedFromSnapshot(snapshot); } else { - await container.createDetached(source.codeDetails); + assert(codeDetails, "One of the source should be there to load from!!"); + await container.createDetached(codeDetails); } + return container; } diff --git a/packages/loader/container-loader/src/loader.ts b/packages/loader/container-loader/src/loader.ts index 222967be5037..f9f71557ad6d 100644 --- a/packages/loader/container-loader/src/loader.ts +++ b/packages/loader/container-loader/src/loader.ts @@ -17,7 +17,7 @@ import { ILoader, IProxyLoaderFactory, LoaderHeader, - DetachedContainerSource, + IFluidCodeDetails, } from "@fluidframework/container-definitions"; import { Deferred, performanceNow } from "@fluidframework/common-utils"; import { ChildLogger, DebugLogger, PerformanceEvent } from "@fluidframework/telemetry-utils"; @@ -27,7 +27,7 @@ import { IResolvedUrl, IUrlResolver, } from "@fluidframework/driver-definitions"; -import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions"; +import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions"; import { ensureFluidResolvedUrl, MultiUrlResolver, @@ -99,10 +99,14 @@ export class RelativeLoader extends EventEmitter implements ILoader { return this.loader.request(request); } - public async createDetachedContainer(source: DetachedContainerSource): Promise { + public async createDetachedContainer(source: IFluidCodeDetails): Promise { throw new Error("Relative loader should not create a detached container"); } + public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { + throw new Error("Relative loader should not create a detached container from snapshot"); + } + public resolveContainer(container: Container) { this.containerDeferred.resolve(container); } @@ -156,7 +160,22 @@ export class Loader extends EventEmitter implements ILoader { this.documentServiceFactory = MultiDocumentServiceFactory.create(documentServiceFactory); } - public async createDetachedContainer(source: DetachedContainerSource): Promise { + public async createDetachedContainer(source: IFluidCodeDetails): Promise { + debug(`Container creating in detached state: ${performanceNow()} `); + + return Container.create( + this.codeLoader, + this.options, + this.scope, + this, + source, + undefined, + this.documentServiceFactory, + this.resolver, + this.subLogger); + } + + public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { debug(`Container creating in detached state: ${performanceNow()} `); return Container.create( @@ -164,6 +183,7 @@ export class Loader extends EventEmitter implements ILoader { this.options, this.scope, this, + undefined, source, this.documentServiceFactory, this.resolver, diff --git a/packages/loader/execution-context-loader/package.json b/packages/loader/execution-context-loader/package.json index 5eebcc1e6420..424b92951ea8 100644 --- a/packages/loader/execution-context-loader/package.json +++ b/packages/loader/execution-context-loader/package.json @@ -30,6 +30,7 @@ "@fluidframework/container-definitions": "^0.25.0", "@fluidframework/core-interfaces": "^0.25.0", "@fluidframework/driver-definitions": "^0.25.0", + "@fluidframework/protocol-definitions": "^0.1011.0-0", "comlink": "^4.0.2", "debug": "^4.1.1" }, diff --git a/packages/loader/execution-context-loader/src/webWorkerLoader.ts b/packages/loader/execution-context-loader/src/webWorkerLoader.ts index bdab53267917..3a7bfaa74471 100644 --- a/packages/loader/execution-context-loader/src/webWorkerLoader.ts +++ b/packages/loader/execution-context-loader/src/webWorkerLoader.ts @@ -9,7 +9,8 @@ import { IRequest, IResponse, } from "@fluidframework/core-interfaces"; -import { IContainer, ILoader, DetachedContainerSource } from "@fluidframework/container-definitions"; +import { IContainer, ILoader, IFluidCodeDetails } from "@fluidframework/container-definitions"; +import { ISnapshotTree } from "@fluidframework/protocol-definitions"; import { IFluidResolvedUrl } from "@fluidframework/driver-definitions"; import Comlink from "comlink"; @@ -71,7 +72,11 @@ export class WebWorkerLoader implements ILoader, IFluidRunnable, IFluidRouter { return this.proxy.resolve(request); } - public async createDetachedContainer(source: DetachedContainerSource): Promise { + public async createDetachedContainer(source: IFluidCodeDetails): Promise { return this.proxy.createDetachedContainer(source); } + + public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { + return this.proxy.createDetachedContainerFromSnapshot(source); + } } diff --git a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts index 76c9e4e77245..f6a0fae04093 100644 --- a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts @@ -9,7 +9,6 @@ import { IFluidCodeDetails, IProxyLoaderFactory, AttachState, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -32,10 +31,7 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const source: DetachedContainerSource = { - codeDetails, - useSnapshot: false, - }; + const mapId1 = "mapId1"; const mapId2 = "mapId2"; @@ -47,7 +43,7 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { `${name} should be ${attached ? "Attached" : "Detached"}`; async function createDetachedContainerAndGetRootComponent() { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(codeDetails); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const defaultComponent = response.value; diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 24d33a27edb9..1ba8491fcd18 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -7,7 +7,6 @@ import assert from "assert"; import { IFluidCodeDetails, IProxyLoaderFactory, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -38,10 +37,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const source: DetachedContainerSource = { - codeDetails, - useSnapshot: false, - }; + const sharedStringId = "ss1Key"; const sharedMapId = "sm1Key"; const crcId = "crc1Key"; @@ -58,7 +54,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { let request: IRequest; async function createDetachedContainerAndGetRootComponent() { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(codeDetails); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const defaultComponent = response.value; @@ -195,12 +191,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { await createDetachedContainerAndGetRootComponent(); const snapshotTree = JSON.parse(container.serialize()); - const rehydrationSource: DetachedContainerSource = { - snapshot: snapshotTree, - useSnapshot: true, - }; - const container2 = await loader.createDetachedContainer(rehydrationSource); + const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); @@ -242,12 +234,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { await createDetachedContainerAndGetRootComponent(); const snapshotTree = JSON.parse(container.serialize()); - const rehydrationSource: DetachedContainerSource = { - snapshot: snapshotTree, - useSnapshot: true, - }; - const container2 = await loader.createDetachedContainer(rehydrationSource); + const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); await container2.attach(request); // Check for scheduler @@ -295,11 +283,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { sharedStringBefore.insertText(0, "Hello"); const snapshotTree = JSON.parse(container.serialize()); - const rehydrationSource: DetachedContainerSource = { - snapshot: snapshotTree, - useSnapshot: true, - }; - const container2 = await loader.createDetachedContainer(rehydrationSource); + + const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseAfter = await container2.request({ url: "/" }); const defaultComponentAfter = responseAfter.value as ITestFluidComponent; @@ -317,12 +302,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); sharedString1.insertText(0, str); const snapshotTree = JSON.parse(container.serialize()); - const rehydrationSource: DetachedContainerSource = { - snapshot: snapshotTree, - useSnapshot: true, - }; - const container2 = await loader.createDetachedContainer(rehydrationSource); + const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseBefore = await container2.request({ url: "/" }); const defaultComponentBefore = responseBefore.value as ITestFluidComponent; const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); diff --git a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts index 16fd4c79f4cc..5c304bb90748 100644 --- a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -9,7 +9,6 @@ import { IFluidCodeDetails, IProxyLoaderFactory, AttachState, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { ConnectionState, Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -41,10 +40,7 @@ describe("Detached Container", () => { package: "detachedContainerTestPackage", config: {}, }; - const source: DetachedContainerSource = { - codeDetails: pkg, - useSnapshot: false, - }; + const sharedStringId = "ss1Key"; const sharedMapId = "sm1Key"; const crcId = "crc1Key"; @@ -99,7 +95,7 @@ describe("Detached Container", () => { }); it("Create detached container", async () => { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); assert.strictEqual(container.attachState, AttachState.Detached, "Container should be detached"); assert.strictEqual(container.closed, false, "Container should be open"); assert.strictEqual(container.deltaManager.inbound.length, 0, "Inbound queue should be empty"); @@ -114,7 +110,7 @@ describe("Detached Container", () => { }); it("Attach detached container", async () => { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); await container.attach(request); assert.strictEqual(container.attachState, AttachState.Attached, "Container should be attached"); assert.strictEqual(container.closed, false, "Container should be open"); @@ -123,7 +119,7 @@ describe("Detached Container", () => { }); it("Components in detached container", async () => { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // Get the root component from the detached container. const response = await container.request({ url: "/" }); if (response.mimeType !== "fluid/object" && response.status !== 200) { @@ -143,7 +139,7 @@ describe("Detached Container", () => { }); it("Components in attached container", async () => { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const component = response.value as ITestFluidComponent; @@ -166,7 +162,7 @@ describe("Detached Container", () => { }); it("Load attached container and check for components", async () => { - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // Get the root component from the detached container. const response = await container.request({ url: "/" }); const component = response.value as ITestFluidComponent; @@ -204,7 +200,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared string", async () => { const ops = { pos1: 0, seg: "b", type: 0 }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.equal(type, MessageType.Operation); @@ -239,7 +235,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared map", async () => { const ops = { key: "1", type: "set", value: { type: "Plain", value: "b" } }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -269,7 +265,7 @@ describe("Detached Container", () => { it("Fire channel attach ops during container attach", async () => { const testChannelId = "testChannel1"; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.id, @@ -298,7 +294,7 @@ describe("Detached Container", () => { it("Fire component attach ops during container attach", async () => { const testComponentType = "default"; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // Get the root component from the detached container. const response = await container.request({ url: "/" }); @@ -329,7 +325,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for consensus register collection", async () => { const op = { key: "1", type: "write", serializedValue: JSON.stringify("b"), refSeq: 0 }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -364,7 +360,7 @@ describe("Detached Container", () => { value: { type: "Plain", value: "b" }, }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -393,7 +389,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared cell", async () => { const op = { type: "setCell", value: { type: "Plain", value: "b" } }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -421,7 +417,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for shared ink", async () => { const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -455,7 +451,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for consensus ordered collection", async () => { const op = { opName: "add", value: JSON.stringify("s") }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -486,7 +482,7 @@ describe("Detached Container", () => { it("Fire ops during container attach for sparse matrix", async () => { const seg = { items: ["s"] }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, @@ -522,7 +518,7 @@ describe("Detached Container", () => { it.skip("Fire ops during container attach for shared matrix", async () => { const op = { pos1: 0, seg: 9, type: 0, target: "rows" }; const defPromise = new Deferred(); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(pkg); // eslint-disable-next-line @typescript-eslint/unbound-method container.deltaManager.submit = (type, contents, batch, metadata) => { assert.strictEqual(contents.contents.contents.content.address, diff --git a/packages/test/service-load-test/src/nodeStressTest.ts b/packages/test/service-load-test/src/nodeStressTest.ts index 9281b70585ec..e7e34b0a6e8e 100644 --- a/packages/test/service-load-test/src/nodeStressTest.ts +++ b/packages/test/service-load-test/src/nodeStressTest.ts @@ -9,7 +9,6 @@ import commander from "commander"; import { IProxyLoaderFactory, IFluidCodeDetails, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { OdspDocumentServiceFactory, OdspDriverUrlResolver } from "@fluidframework/odsp-driver"; @@ -42,11 +41,6 @@ const codeDetails: IFluidCodeDetails = { config: {}, }; -const source: DetachedContainerSource = { - codeDetails, - useSnapshot: false, -}; - const codeLoader = new LocalCodeLoader([[codeDetails, fluidExport]]); const urlResolver = new OdspDriverUrlResolver(); const odspTokenManager = new OdspTokenManager(odspTokensCache); @@ -91,7 +85,7 @@ function createLoader(config: IConfig, password: string) { async function initialize(config: IConfig, password: string) { const loader = createLoader(config, password); - const container = await loader.createDetachedContainer(source); + const container = await loader.createDetachedContainer(codeDetails); container.on("error", (error) => { console.log(error); process.exit(-1); diff --git a/packages/tools/webpack-component-loader/src/loader.ts b/packages/tools/webpack-component-loader/src/loader.ts index ee1ca921910b..e1ec6de7c543 100644 --- a/packages/tools/webpack-component-loader/src/loader.ts +++ b/packages/tools/webpack-component-loader/src/loader.ts @@ -15,7 +15,6 @@ import { IProxyLoaderFactory, IResolvedFluidCodeDetails, isFluidPackage, - DetachedContainerSource, } from "@fluidframework/container-definitions"; import { Container, Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; @@ -189,11 +188,6 @@ export async function start( config: {}, }; - const source: DetachedContainerSource = { - codeDetails, - useSnapshot: false, - }; - const urlResolver = new MultiUrlResolver(window.location.origin, options); // Create the loader that is used to load the Container. @@ -202,7 +196,7 @@ export async function start( let container1: Container; if (autoAttach || manualAttach) { // For new documents, create a detached container which will be attached later. - container1 = await loader1.createDetachedContainer(source); + container1 = await loader1.createDetachedContainer(codeDetails); } else { // For existing documents, we try to load the container with the given documentId. const documentUrl = `${window.location.origin}/${documentId}`; @@ -221,7 +215,7 @@ export async function start( url = url.replace(id, documentId); const newLoader = await createWebLoader(documentId, fluidModule, options, urlResolver, codeDetails); - container1 = await newLoader.createDetachedContainer(source); + container1 = await newLoader.createDetachedContainer(codeDetails); } } From 60f977f131d5b1a892f5c133d3ac4f6f29aa64bb Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 12 Aug 2020 12:53:14 -0700 Subject: [PATCH 16/27] resolve m,erge --- .../src/getTinyliciousContainer.ts | 7 +- .../src/channel.ts | 187 ------------------ .../src/test/detachedContainerTests.spec.ts | 6 +- .../service-load-test/src/nodeStressTest.ts | 5 +- 4 files changed, 6 insertions(+), 199 deletions(-) delete mode 100644 packages/runtime/component-runtime-definitions/src/channel.ts diff --git a/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts b/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts index d9ea6833f19e..b561aa786e40 100644 --- a/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts +++ b/examples/utils/get-tinylicious-container/src/getTinyliciousContainer.ts @@ -4,7 +4,9 @@ */ import { IRequest } from "@fluidframework/core-interfaces"; -import { IRuntimeFactory } from "@fluidframework/container-definitions"; +import { + IRuntimeFactory, +} from "@fluidframework/container-definitions"; import { Container, Loader } from "@fluidframework/container-loader"; import { IFluidResolvedUrl, @@ -98,8 +100,7 @@ export async function getTinyliciousContainer( // We're not actually using the code proposal (our code loader always loads the same module regardless of the // proposal), but the Container will only give us a NullRuntime if there's no proposal. So we'll use a fake // proposal. - const codeDetails = { package: "", config: {} }; - container = await loader.createDetachedContainer(codeDetails); + container = await loader.createDetachedContainer({ package: "", config: {} }); await container.attach({ url: documentId }); } else { // The InsecureTinyliciousUrlResolver expects the url of the request to be the documentId. diff --git a/packages/runtime/component-runtime-definitions/src/channel.ts b/packages/runtime/component-runtime-definitions/src/channel.ts deleted file mode 100644 index 60a1cac6c34e..000000000000 --- a/packages/runtime/component-runtime-definitions/src/channel.ts +++ /dev/null @@ -1,187 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import { IFluidLoadable } from "@fluidframework/component-core-interfaces"; -import { ISequencedDocumentMessage, ITree } from "@fluidframework/protocol-definitions"; -import { IChannelAttributes } from "./storage"; -import { IFluidDataStoreRuntime } from "./componentRuntime"; - -declare module "@fluidframework/component-core-interfaces" { - // eslint-disable-next-line @typescript-eslint/no-empty-interface - interface IComponent extends Readonly> { } - - // eslint-disable-next-line @typescript-eslint/no-empty-interface - interface IFluidObject extends Readonly> { } -} - -export const IChannel: keyof IProvideChannel = "IChannel"; - -export interface IProvideChannel { - readonly IChannel: IChannel; -} - -export interface IChannel extends IProvideChannel, IFluidLoadable { - /** - * A readonly identifier for the channel - */ - readonly id: string; - - readonly owner?: string; - - readonly attributes: IChannelAttributes; - - /** - * Generates snapshot of the channel. - */ - snapshot(): ITree; - - /** - * True if the data structure is attached to storage. - */ - isAttached(): boolean; - - /** - * Enables the channel to send and receive ops - */ - connect(services: IChannelServices): void; -} - -/** - * Handler provided by shared data structure to process requests from the runtime. - */ -export interface IDeltaHandler { - /** - * Processes the op. - * @param message - The message to process - * @param local - Whether the message originated from the local client - * @param localOpMetadata - For local client messages, this is the metadata that was submitted with the message. - * For messages from a remote client, this will be undefined. - */ - process: (message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown) => void; - - /** - * State change events to indicate changes to the delta connection - * @param connected - true if connected, false otherwise - */ - setConnectionState(connected: boolean): void; - - /** - * Called when the runtime asks the client to resubmit an op. This may be because the Container reconnected and - * this op was not acked. - * The client can choose to resubmit the same message, submit different / multiple messages or not submit anything - * at all. - * @param message - The original message that was submitted. - * @param localOpMetadata - The local metadata associated with the original message. - */ - reSubmit(message: any, localOpMetadata: unknown): void; -} - -/** - * Interface to represent a connection to a delta notification stream. - */ -export interface IDeltaConnection { - connected: boolean; - - /** - * Send new messages to the server. - * @param messageContent - The content of the message to be sent. - * @param localOpMetadata - The local metadata associated with the message. This is kept locally by the runtime - * and not sent to the server. It will be provided back when this message is acknowledged by the server. It will - * also be provided back when asked to resubmit the message. - */ - submit(messageContent: any, localOpMetadata: unknown): void; - - /** - * Attaches a message handler to the delta connection - */ - attach(handler: IDeltaHandler): void; - - /** - * Indicates that the channel is dirty and needs to be part of the summary. It is called by a SharedSummaryBlock - * that needs to be part of the summary but does not generate ops. - */ - dirty(): void; -} - -/** - * Storage services to read the objects at a given path. - */ -export interface IChannelStorageService { - /** - * Reads the object contained at the given path. Returns a base64 string representation for the object. - */ - read(path: string): Promise; - - /** - * Determines if there is an object contained at the given path. - */ - contains(path: string): Promise; - - /** - * Lists the blobs that exist at a specific path. - */ - list(path: string): Promise; -} - -/** - * Storage services to read the objects at a given path using the given delta connection. - */ -export interface IChannelServices { - deltaConnection: IDeltaConnection; - - objectStorage: IChannelStorageService; -} - -/** - * Definitions of a channel factory. Factories follow a common model but enable custom behavior. - */ -export interface IChannelFactory { - /** - * String representing the type of the factory. - */ - readonly type: string; - - /** - * Attributes of the channel. - */ - readonly attributes: IChannelAttributes; - - /** - * Loads the given channel. This call is only ever invoked internally as the only thing - * that is ever directly loaded is the document itself. Load will then only be called on documents that - * were created and added to a channel. - * @param runtime - Component runtime containing state/info/helper methods about the component. - * @param id - ID of the channel. - * @param services - Services to read objects at a given path using the delta connection. - * @param branchId - The branch ID. - * @param channelAttributes - The attributes for the the channel to be loaded. - * @returns The loaded object - * - * @privateRemarks - * Thought: should the storage object include the version information and limit access to just files - * for the given object? The latter seems good in general. But both are probably good things. We then just - * need a way to allow the document to provide later storage for the object. - */ - load( - runtime: IFluidDataStoreRuntime, - id: string, - services: IChannelServices, - branchId: string, - channelAttributes: Readonly, - ): Promise; - - /** - * Creates a local version of the channel. - * Calling attach on the object later will insert it into the object stream. - * @param runtime - The runtime the new object will be associated with - * @param id - The unique ID of the new object - * @returns The newly created object. - * - * @privateRemarks - * NOTE here - When we attach we need to submit all the pending ops prior to actually doing the attach - * for consistency. - */ - create(runtime: IFluidDataStoreRuntime, id: string): IChannel; -} diff --git a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts index 5c304bb90748..378fe3d91d67 100644 --- a/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -5,11 +5,7 @@ import assert from "assert"; import { IRequest } from "@fluidframework/core-interfaces"; -import { - IFluidCodeDetails, - IProxyLoaderFactory, - AttachState, -} from "@fluidframework/container-definitions"; +import { IFluidCodeDetails, IProxyLoaderFactory, AttachState } from "@fluidframework/container-definitions"; import { ConnectionState, Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; diff --git a/packages/test/service-load-test/src/nodeStressTest.ts b/packages/test/service-load-test/src/nodeStressTest.ts index e7e34b0a6e8e..3328773d9893 100644 --- a/packages/test/service-load-test/src/nodeStressTest.ts +++ b/packages/test/service-load-test/src/nodeStressTest.ts @@ -6,10 +6,7 @@ import fs from "fs"; import child_process from "child_process"; import commander from "commander"; -import { - IProxyLoaderFactory, - IFluidCodeDetails, -} from "@fluidframework/container-definitions"; +import { IProxyLoaderFactory, IFluidCodeDetails } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { OdspDocumentServiceFactory, OdspDriverUrlResolver } from "@fluidframework/odsp-driver"; import { LocalCodeLoader } from "@fluidframework/test-utils"; From f93b1df7d494a7e3c3d34eb3be7cec43a91dea6c Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Mon, 17 Aug 2020 00:11:37 -0700 Subject: [PATCH 17/27] set existing as true if loaded from rehydration snapshopt --- .../baseContainerRuntimeFactory.ts | 2 +- .../loader/container-loader/src/container.ts | 8 ++- .../runtime/agent-scheduler/src/scheduler.ts | 2 +- .../container-runtime/src/containerRuntime.ts | 34 ++++++---- .../container-runtime/src/dataStoreContext.ts | 5 +- .../src/dataStoreRuntime.ts | 2 - .../runtime/datastore/src/dataStoreRuntime.ts | 39 +++++------ .../src/dataStoreContext.ts | 6 ++ .../runtime/test-runtime-utils/src/mocks.ts | 4 -- .../test/test-utils/src/testFluidComponent.ts | 2 +- .../tools/webpack-fluid-loader/src/loader.ts | 67 ++++++++++++++++--- 11 files changed, 114 insertions(+), 57 deletions(-) diff --git a/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts b/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts index c1ddabb4fc4b..0fd74bf97f62 100644 --- a/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts +++ b/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts @@ -78,7 +78,7 @@ export class BaseContainerRuntimeFactory implements dc.register(IContainerRuntime, runtime); // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). - if (!(runtime.existing || context.baseSnapshot)) { + if (!runtime.existing) { // If it's the first time through. await this.containerInitializingFirstTime(runtime); } diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index a4ec53afbe68..828b7fd50915 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -544,8 +544,6 @@ export class Container extends EventEmitterWithErrorHandling i this.blobManager = await this.loadBlobManager(this.storageService, undefined); this._attachState = AttachState.Attached; this.emit("attached"); - // We know this is create new flow. - this._existing = false; this._parentBranch = this._id; // Propagate current connection state through the system. @@ -990,6 +988,9 @@ export class Container extends EventEmitterWithErrorHandling i this.attachDeltaManagerOpHandler(attributes); + // We know this is create detached flow without snapshot. + this._existing = false; + // Need to just seed the source data in the code quorum. Quorum itself is empty this.protocolHandler = this.initializeProtocolState( attributes, @@ -1010,6 +1011,9 @@ export class Container extends EventEmitterWithErrorHandling i assert.strictEqual(attributes.sequenceNumber, 0, "Seq number in detached container should be 0!!"); this.attachDeltaManagerOpHandler(attributes); + // We know this is create detached flow with snapshot. + this._existing = true; + // ...load in the existing quorum // Initialize the protocol handler this.protocolHandler = diff --git a/packages/runtime/agent-scheduler/src/scheduler.ts b/packages/runtime/agent-scheduler/src/scheduler.ts index 123c05197917..f85251f79bbc 100644 --- a/packages/runtime/agent-scheduler/src/scheduler.ts +++ b/packages/runtime/agent-scheduler/src/scheduler.ts @@ -37,7 +37,7 @@ class AgentScheduler extends EventEmitter implements IAgentScheduler, IFluidRout let root: ISharedMap; let scheduler: ConsensusRegisterCollection; // Don't create if existing on storage or loaded detached from snapshot(ex. draft mode). - if (!(runtime.existing || runtime.isLoadedFromSnapshot)) { + if (!runtime.existing) { root = SharedMap.create(runtime, "root"); root.bindToContext(); scheduler = ConsensusRegisterCollection.create(runtime); diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index b9a222ba718d..b194ab1056b2 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -89,6 +89,7 @@ import { CreateSummarizerNodeSource, IAgentScheduler, ITaskManager, + ISummarizeResult, } from "@fluidframework/runtime-definitions"; import { FluidSerializer, @@ -497,7 +498,7 @@ export class ContainerRuntime extends EventEmitter // Create all internal stores if not already existing on storage or loaded a detached // container from snapshot(ex. draft mode). - if (!(context.existing || context.baseSnapshot)) { + if (!context.existing) { await runtime.createRootDataStore(schedulerId, schedulerId); } @@ -586,6 +587,10 @@ export class ContainerRuntime extends EventEmitter return (this.context as any).isAttached() ? AttachState.Attached : AttachState.Detached; } + public isLocalDataStore(id: string): boolean { + return this.localContexts.has(id); + } + public nextSummarizerP?: Promise; public nextSummarizerD?: Deferred; @@ -660,6 +665,7 @@ export class ContainerRuntime extends EventEmitter // Attached and loaded context proxies private readonly contexts = new Map(); + private readonly localContexts = new Set(); // List of pending contexts (for the case where a client knows a store will exist and is waiting // on its creation). This is a superset of contexts. private readonly contextsDeferred = new Map>(); @@ -758,8 +764,17 @@ export class ContainerRuntime extends EventEmitter // Create a context for each of them for (const [key, value] of fluidDataStores) { let dataStoreContext: FluidDataStoreContext; - // If it is loaded from a snapshot but in detached state, then create a local component. - if (this.attachState === AttachState.Detached) { + // If we have a detached container, then create local data store contexts. + if (this.attachState !== AttachState.Detached) { + dataStoreContext = new RemotedFluidDataStoreContext( + key, + typeof value === "string" ? value : Promise.resolve(value), + this, + this.storage, + this.containerScope, + this.summaryTracker.createOrGetChild(key, this.summaryTracker.referenceSequenceNumber), + this.summarizerNode.getCreateChildFn(key, { type: CreateSummarizerNodeSource.FromSummary })); + } else { let pkgFromSnapshot: string[]; // back-compat 0.24 baseSnapshotCouldBeNull if (context.baseSnapshot === null || context.baseSnapshot === undefined) { @@ -777,6 +792,7 @@ export class ContainerRuntime extends EventEmitter } else { throw new Error(`Invalid snapshot format version ${snapshotFormatVersion}`); } + this.localContexts.add(key); dataStoreContext = new LocalFluidDataStoreContext( key, pkgFromSnapshot, @@ -787,15 +803,6 @@ export class ContainerRuntime extends EventEmitter this.summarizerNode.getCreateChildFn(key, { type: CreateSummarizerNodeSource.FromSummary }), (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr), snapshotTree); - } else { - dataStoreContext = new RemotedFluidDataStoreContext( - key, - typeof value === "string" ? value : Promise.resolve(value), - this, - this.storage, - this.containerScope, - this.summaryTracker.createOrGetChild(key, this.summaryTracker.referenceSequenceNumber), - this.summarizerNode.getCreateChildFn(key, { type: CreateSummarizerNodeSource.FromSummary })); } this.setNewContext(key, dataStoreContext); } @@ -1297,6 +1304,7 @@ export class ContainerRuntime extends EventEmitter const deferred = new Deferred(); this.contextsDeferred.set(id, deferred); + this.localContexts.add(id); this.contexts.set(id, context); return context; @@ -1324,7 +1332,7 @@ export class ContainerRuntime extends EventEmitter const deferred = new Deferred(); this.contextsDeferred.set(id, deferred); this.contexts.set(id, context); - + this.localContexts.add(id); if (realizationFn) { return context.realizeWithFn(realizationFn); } else { diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index e377ed751b13..80be1a46a869 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -202,7 +202,8 @@ export abstract class FluidDataStoreContext extends EventEmitter implements ) { super(); - this._attachState = existing ? AttachState.Attached : AttachState.Detached; + this._attachState = this.containerRuntime.attachState !== AttachState.Detached && existing ? + this.containerRuntime.attachState : AttachState.Detached; this.bindToContext = (channel: IFluidDataStoreChannel) => { assert(this.bindState === BindState.NotBound); @@ -705,7 +706,7 @@ export class LocalFluidDataStoreContext extends FluidDataStoreContext { super( runtime, id, - false, + snapshotTree !== undefined ? true : false, storage, scope, summaryTracker, diff --git a/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts b/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts index d013c2124a11..edeef5fe1bef 100644 --- a/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore-definitions/src/dataStoreRuntime.ts @@ -51,8 +51,6 @@ export interface IFluidDataStoreRuntime extends readonly existing: boolean; - readonly isLoadedFromSnapshot: boolean; - readonly parentBranch: string | null; readonly connected: boolean; diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index 0303e4b25ed5..9a71f4365ed2 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -143,10 +143,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto return this._attachState; } - public get isLoadedFromSnapshot(): boolean { - return this.dataStoreContext.baseSnapshot !== undefined; - } - /** * @deprecated - 0.21 back-compat */ @@ -207,9 +203,21 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto if (tree?.trees !== undefined) { Object.keys(tree.trees).forEach((path) => { let channelContext: IChannelContext; - // If already exists on storage, then create a remote channel. However, if it is loaded from a snpashot - // but not yet exists on storage, then create a Local Channel. - if (existing) { + // If already exists on storage, then create a remote channel. + if (dataStoreContext.containerRuntime.isLocalDataStore(id)) { + const channelAttributes = readAndParseFromBlobs( + tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); + channelContext = new LocalChannelContext( + path, + this.sharedObjectRegistry, + channelAttributes.type, + this, + this.dataStoreContext, + this.dataStoreContext.storage, + (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), + (address: string) => this.setChannelDirty(address), + tree.trees[path]); + } else { channelContext = new RemoteChannelContext( this, dataStoreContext, @@ -229,19 +237,6 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto path, { type: CreateSummarizerNodeSource.FromSummary }, )); - } else { - const channelAttributes = readAndParseFromBlobs( - tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); - channelContext = new LocalChannelContext( - path, - this.sharedObjectRegistry, - channelAttributes.type, - this, - this.dataStoreContext, - this.dataStoreContext.storage, - (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata), - (address: string) => this.setChannelDirty(address), - tree.trees[path]); } const deferred = new Deferred(); deferred.resolve(channelContext); @@ -253,8 +248,8 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto this.attachListener(); // If exists on storage or loaded from a snapshot, it should already be binded. - this.bindState = existing || tree !== undefined ? BindState.Bound : BindState.NotBound; - this._attachState = existing ? AttachState.Attached : AttachState.Detached; + this.bindState = existing ? BindState.Bound : BindState.NotBound; + this._attachState = dataStoreContext.attachState; // If it's existing we know it has been attached. if (existing) { diff --git a/packages/runtime/runtime-definitions/src/dataStoreContext.ts b/packages/runtime/runtime-definitions/src/dataStoreContext.ts index 581f3aaa6f9f..f3593933e9d1 100644 --- a/packages/runtime/runtime-definitions/src/dataStoreContext.ts +++ b/packages/runtime/runtime-definitions/src/dataStoreContext.ts @@ -117,6 +117,12 @@ export interface IContainerRuntimeBase extends getAbsoluteUrl(relativeUrl: string): Promise; getTaskManager(): Promise; + + /** + * True if the data store with given id is created by this client. + * @param id - id of the data store. + */ + isLocalDataStore(id: string): boolean; } /** diff --git a/packages/runtime/test-runtime-utils/src/mocks.ts b/packages/runtime/test-runtime-utils/src/mocks.ts index 05f493fdbb8f..5dd9ded141d1 100644 --- a/packages/runtime/test-runtime-utils/src/mocks.ts +++ b/packages/runtime/test-runtime-utils/src/mocks.ts @@ -395,10 +395,6 @@ export class MockFluidDataStoreRuntime extends EventEmitter this._local = local; } - public get isLoadedFromSnapshot() { - return false; - } - private _disposed = false; public get disposed() { return this._disposed; } diff --git a/packages/test/test-utils/src/testFluidComponent.ts b/packages/test/test-utils/src/testFluidComponent.ts index ccf47e946cf2..7029961de83e 100644 --- a/packages/test/test-utils/src/testFluidComponent.ts +++ b/packages/test/test-utils/src/testFluidComponent.ts @@ -91,7 +91,7 @@ export class TestFluidComponent implements ITestFluidComponent { private async initialize() { // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). - if (!(this.runtime.existing || this.runtime.isLoadedFromSnapshot)) { + if (!this.runtime.existing) { this.root = SharedMap.create(this.runtime, "root"); this.factoryEntriesMap.forEach((sharedObjectFactory: IChannelFactory, key: string) => { diff --git a/packages/tools/webpack-fluid-loader/src/loader.ts b/packages/tools/webpack-fluid-loader/src/loader.ts index ee77e180b60a..634f03b58186 100644 --- a/packages/tools/webpack-fluid-loader/src/loader.ts +++ b/packages/tools/webpack-fluid-loader/src/loader.ts @@ -245,7 +245,18 @@ export async function start( // We have rendered the fluid object. If the container is detached, attach it now. if (container1.attachState === AttachState.Detached) { - await attachContainer(container1, urlResolver, documentId, url, rightDiv, manualAttach); + await attachContainer( + loader1, + container1, + fluidObjectUrl, + urlResolver, + documentId, + url, + div, + leftDiv, + rightDiv, + manualAttach, + ); } // For side by side mode, we need to create a second container and fluid object. @@ -308,11 +319,15 @@ async function getFluidObjectAndRender(container: Container, url: string, div: H * is clicked. Otherwise, it attaches the conatiner right away. */ async function attachContainer( + loader: Loader, container: Container, + fluidObjectUrl: string, urlResolver: MultiUrlResolver, documentId: string, url: string, - div: HTMLDivElement | undefined, + mainDiv: HTMLDivElement, + leftDiv: HTMLDivElement, + rightDiv: HTMLDivElement | undefined, manualAttach: boolean, ) { // This is called once loading is complete to replace the url in the address bar with the new `url`. @@ -321,6 +336,7 @@ async function attachContainer( document.title = documentId; }; + let currentContainer = container; const attached = new Deferred(); const attachUrl = await urlResolver.createRequestForCreateNew(documentId); @@ -329,17 +345,50 @@ async function attachContainer( const attachDiv = document.createElement("div"); const attachButton = document.createElement("button"); attachButton.innerText = "Attach Container"; + const serializeButton = document.createElement("button"); + serializeButton.innerText = "Serialize"; + const rehydrateButton = document.createElement("button"); + rehydrateButton.innerText = "Rehydrate Container"; + rehydrateButton.hidden = true; + const summaryList = document.createElement("select"); + summaryList.hidden = true; attachDiv.append(attachButton); + attachDiv.append(serializeButton); + attachDiv.append(summaryList); document.body.prepend(attachDiv); + let summaryNum = 1; + serializeButton.onclick = () => { + summaryList.hidden = false; + rehydrateButton.hidden = false; + attachDiv.append(rehydrateButton); + const summary = currentContainer.serialize(); + const listItem = document.createElement("option"); + listItem.innerText = `Summary_${summaryNum}`; + summaryNum += 1; + listItem.value = summary; + summaryList.appendChild(listItem); + rehydrateButton.onclick = async () => { + const snapshot = summaryList.value; + currentContainer = await loader.createDetachedContainerFromSnapshot(JSON.parse(snapshot)); + currentContainer.removeAllListeners(); + // Load and render the component. + await getFluidObjectAndRender(currentContainer, fluidObjectUrl, rightDiv); + // Handle the code upgrade scenario (which fires contextChanged) + currentContainer.on("contextChanged", () => { + getFluidObjectAndRender(currentContainer, fluidObjectUrl, rightDiv).catch(() => { }); + }); + }; + }; + attachButton.onclick = () => { - container.attach(attachUrl) + currentContainer.attach(attachUrl) .then(() => { attachDiv.remove(); replaceUrl(); - if (div) { - div.innerText = ""; + if (rightDiv) { + rightDiv.innerText = ""; } attached.resolve(); @@ -349,14 +398,14 @@ async function attachContainer( }; // If we are in side-by-side mode, we need to display the following message in the right div passed here. - if (div) { - div.innerText = "Waiting for container attach"; + if (rightDiv) { + rightDiv.innerText = "Waiting for container attach"; } } else { - await container.attach(attachUrl); + await currentContainer.attach(attachUrl); replaceUrl(); attached.resolve(); } - return attached.promise; + return Promise.all([attached.promise, Promise.resolve(currentContainer)]); } From 79592b12e53bdfa8cb11cad5cb6451385e10e08a Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Mon, 17 Aug 2020 00:13:51 -0700 Subject: [PATCH 18/27] set existing as true if loaded from rehydration snapshopt --- packages/runtime/runtime-definitions/src/dataStoreContext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/runtime-definitions/src/dataStoreContext.ts b/packages/runtime/runtime-definitions/src/dataStoreContext.ts index f3593933e9d1..220ab6eebfda 100644 --- a/packages/runtime/runtime-definitions/src/dataStoreContext.ts +++ b/packages/runtime/runtime-definitions/src/dataStoreContext.ts @@ -119,7 +119,7 @@ export interface IContainerRuntimeBase extends getTaskManager(): Promise; /** - * True if the data store with given id is created by this client. + * True if the data store with given id is created by this container instance. * @param id - id of the data store. */ isLocalDataStore(id: string): boolean; From acd860df6230c75130cc31e352c635466a171711 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Mon, 17 Aug 2020 00:22:34 -0700 Subject: [PATCH 19/27] remove --- .../container-runtime-factories/baseContainerRuntimeFactory.ts | 1 - packages/runtime/agent-scheduler/src/scheduler.ts | 1 - packages/test/test-utils/src/testFluidComponent.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts b/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts index 0fd74bf97f62..13d05ee3da93 100644 --- a/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts +++ b/packages/framework/aqueduct/src/container-runtime-factories/baseContainerRuntimeFactory.ts @@ -77,7 +77,6 @@ export class BaseContainerRuntimeFactory implements // we register the runtime so developers of providers can use it in the factory pattern. dc.register(IContainerRuntime, runtime); - // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). if (!runtime.existing) { // If it's the first time through. await this.containerInitializingFirstTime(runtime); diff --git a/packages/runtime/agent-scheduler/src/scheduler.ts b/packages/runtime/agent-scheduler/src/scheduler.ts index a5ada429f732..0f68a13b1251 100644 --- a/packages/runtime/agent-scheduler/src/scheduler.ts +++ b/packages/runtime/agent-scheduler/src/scheduler.ts @@ -36,7 +36,6 @@ class AgentScheduler extends EventEmitter implements IAgentScheduler, IFluidRout public static async load(runtime: IFluidDataStoreRuntime, context: IFluidDataStoreContext) { let root: ISharedMap; let scheduler: ConsensusRegisterCollection; - // Don't create if existing on storage or loaded detached from snapshot(ex. draft mode). if (!runtime.existing) { root = SharedMap.create(runtime, "root"); root.bindToContext(); diff --git a/packages/test/test-utils/src/testFluidComponent.ts b/packages/test/test-utils/src/testFluidComponent.ts index 76c7d4c711f3..e7cf24a48518 100644 --- a/packages/test/test-utils/src/testFluidComponent.ts +++ b/packages/test/test-utils/src/testFluidComponent.ts @@ -90,7 +90,6 @@ export class TestFluidComponent implements ITestFluidComponent { } private async initialize() { - // Don't initialize if existing on storage or loaded detached from snapshot(ex. draft mode). if (!this.runtime.existing) { this.root = SharedMap.create(this.runtime, "root"); From e52d2ac0ac5d2f50e6ecf1b96b29fba12937a32a Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Mon, 17 Aug 2020 00:26:26 -0700 Subject: [PATCH 20/27] remove --- .../tools/webpack-fluid-loader/src/loader.ts | 67 +++---------------- 1 file changed, 9 insertions(+), 58 deletions(-) diff --git a/packages/tools/webpack-fluid-loader/src/loader.ts b/packages/tools/webpack-fluid-loader/src/loader.ts index 634f03b58186..ee77e180b60a 100644 --- a/packages/tools/webpack-fluid-loader/src/loader.ts +++ b/packages/tools/webpack-fluid-loader/src/loader.ts @@ -245,18 +245,7 @@ export async function start( // We have rendered the fluid object. If the container is detached, attach it now. if (container1.attachState === AttachState.Detached) { - await attachContainer( - loader1, - container1, - fluidObjectUrl, - urlResolver, - documentId, - url, - div, - leftDiv, - rightDiv, - manualAttach, - ); + await attachContainer(container1, urlResolver, documentId, url, rightDiv, manualAttach); } // For side by side mode, we need to create a second container and fluid object. @@ -319,15 +308,11 @@ async function getFluidObjectAndRender(container: Container, url: string, div: H * is clicked. Otherwise, it attaches the conatiner right away. */ async function attachContainer( - loader: Loader, container: Container, - fluidObjectUrl: string, urlResolver: MultiUrlResolver, documentId: string, url: string, - mainDiv: HTMLDivElement, - leftDiv: HTMLDivElement, - rightDiv: HTMLDivElement | undefined, + div: HTMLDivElement | undefined, manualAttach: boolean, ) { // This is called once loading is complete to replace the url in the address bar with the new `url`. @@ -336,7 +321,6 @@ async function attachContainer( document.title = documentId; }; - let currentContainer = container; const attached = new Deferred(); const attachUrl = await urlResolver.createRequestForCreateNew(documentId); @@ -345,50 +329,17 @@ async function attachContainer( const attachDiv = document.createElement("div"); const attachButton = document.createElement("button"); attachButton.innerText = "Attach Container"; - const serializeButton = document.createElement("button"); - serializeButton.innerText = "Serialize"; - const rehydrateButton = document.createElement("button"); - rehydrateButton.innerText = "Rehydrate Container"; - rehydrateButton.hidden = true; - const summaryList = document.createElement("select"); - summaryList.hidden = true; attachDiv.append(attachButton); - attachDiv.append(serializeButton); - attachDiv.append(summaryList); document.body.prepend(attachDiv); - let summaryNum = 1; - serializeButton.onclick = () => { - summaryList.hidden = false; - rehydrateButton.hidden = false; - attachDiv.append(rehydrateButton); - const summary = currentContainer.serialize(); - const listItem = document.createElement("option"); - listItem.innerText = `Summary_${summaryNum}`; - summaryNum += 1; - listItem.value = summary; - summaryList.appendChild(listItem); - rehydrateButton.onclick = async () => { - const snapshot = summaryList.value; - currentContainer = await loader.createDetachedContainerFromSnapshot(JSON.parse(snapshot)); - currentContainer.removeAllListeners(); - // Load and render the component. - await getFluidObjectAndRender(currentContainer, fluidObjectUrl, rightDiv); - // Handle the code upgrade scenario (which fires contextChanged) - currentContainer.on("contextChanged", () => { - getFluidObjectAndRender(currentContainer, fluidObjectUrl, rightDiv).catch(() => { }); - }); - }; - }; - attachButton.onclick = () => { - currentContainer.attach(attachUrl) + container.attach(attachUrl) .then(() => { attachDiv.remove(); replaceUrl(); - if (rightDiv) { - rightDiv.innerText = ""; + if (div) { + div.innerText = ""; } attached.resolve(); @@ -398,14 +349,14 @@ async function attachContainer( }; // If we are in side-by-side mode, we need to display the following message in the right div passed here. - if (rightDiv) { - rightDiv.innerText = "Waiting for container attach"; + if (div) { + div.innerText = "Waiting for container attach"; } } else { - await currentContainer.attach(attachUrl); + await container.attach(attachUrl); replaceUrl(); attached.resolve(); } - return Promise.all([attached.promise, Promise.resolve(currentContainer)]); + return attached.promise; } From 3f46d226b69f44d2f3e36e290ef5e10c4da0b4f5 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Mon, 17 Aug 2020 01:13:08 -0700 Subject: [PATCH 21/27] localdatastore --- .../runtime/container-runtime/src/containerRuntime.ts | 9 +-------- .../runtime/container-runtime/src/dataStoreContext.ts | 3 +++ packages/runtime/datastore/src/dataStoreRuntime.ts | 2 +- .../runtime/runtime-definitions/src/dataStoreContext.ts | 8 ++------ 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index b194ab1056b2..d126fe70dda0 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -587,10 +587,6 @@ export class ContainerRuntime extends EventEmitter return (this.context as any).isAttached() ? AttachState.Attached : AttachState.Detached; } - public isLocalDataStore(id: string): boolean { - return this.localContexts.has(id); - } - public nextSummarizerP?: Promise; public nextSummarizerD?: Deferred; @@ -665,7 +661,6 @@ export class ContainerRuntime extends EventEmitter // Attached and loaded context proxies private readonly contexts = new Map(); - private readonly localContexts = new Set(); // List of pending contexts (for the case where a client knows a store will exist and is waiting // on its creation). This is a superset of contexts. private readonly contextsDeferred = new Map>(); @@ -792,7 +787,6 @@ export class ContainerRuntime extends EventEmitter } else { throw new Error(`Invalid snapshot format version ${snapshotFormatVersion}`); } - this.localContexts.add(key); dataStoreContext = new LocalFluidDataStoreContext( key, pkgFromSnapshot, @@ -1304,7 +1298,6 @@ export class ContainerRuntime extends EventEmitter const deferred = new Deferred(); this.contextsDeferred.set(id, deferred); - this.localContexts.add(id); this.contexts.set(id, context); return context; @@ -1332,7 +1325,7 @@ export class ContainerRuntime extends EventEmitter const deferred = new Deferred(); this.contextsDeferred.set(id, deferred); this.contexts.set(id, context); - this.localContexts.add(id); + if (realizationFn) { return context.realizeWithFn(realizationFn); } else { diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 80be1a46a869..5496f6c848d4 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -197,6 +197,7 @@ export abstract class FluidDataStoreContext extends EventEmitter implements public readonly summaryTracker: SummaryTracker, createSummarizerNode: CreateChildSummarizerNodeFn, private bindState: BindState, + public readonly isLocalDataStore: boolean, bindChannel: (channel: IFluidDataStoreChannel) => void, protected pkg?: readonly string[], ) { @@ -622,6 +623,7 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext { summaryTracker, createSummarizerNode, BindState.Bound, + false, () => { throw new Error("Already attached"); }, @@ -712,6 +714,7 @@ export class LocalFluidDataStoreContext extends FluidDataStoreContext { summaryTracker, createSummarizerNode, snapshotTree ? BindState.Bound : BindState.NotBound, + true, bindChannel, pkg); this.pkg = pkg; // TODO: avoid setting twice diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index 9a71f4365ed2..997438b9b53d 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -204,7 +204,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto Object.keys(tree.trees).forEach((path) => { let channelContext: IChannelContext; // If already exists on storage, then create a remote channel. - if (dataStoreContext.containerRuntime.isLocalDataStore(id)) { + if (dataStoreContext.isLocalDataStore) { const channelAttributes = readAndParseFromBlobs( tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); channelContext = new LocalChannelContext( diff --git a/packages/runtime/runtime-definitions/src/dataStoreContext.ts b/packages/runtime/runtime-definitions/src/dataStoreContext.ts index 220ab6eebfda..91c000258eff 100644 --- a/packages/runtime/runtime-definitions/src/dataStoreContext.ts +++ b/packages/runtime/runtime-definitions/src/dataStoreContext.ts @@ -117,12 +117,6 @@ export interface IContainerRuntimeBase extends getAbsoluteUrl(relativeUrl: string): Promise; getTaskManager(): Promise; - - /** - * True if the data store with given id is created by this container instance. - * @param id - id of the data store. - */ - isLocalDataStore(id: string): boolean; } /** @@ -260,6 +254,8 @@ export interface IFluidDataStoreContext extends EventEmitter { readonly branch: string; readonly baseSnapshot: ISnapshotTree | undefined; readonly loader: ILoader; + // True if the data store is LocalDataStoreContext. + readonly isLocalDataStore: boolean; /** * Indicates the attachment state of the data store to a host service. */ From a19116fb0f42a3cc60ebfff50994bd352e354c52 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 19 Aug 2020 16:05:15 -0700 Subject: [PATCH 22/27] rename comp toi datta store --- .../container-runtime/src/containerRuntime.ts | 4 +- .../datastore/src/localChannelContext.ts | 6 +- .../test/attachRegisterLocalApiTests.spec.ts | 7 +- .../test/deRehydrateContainerTests.spec.ts | 108 +++++++++--------- 4 files changed, 60 insertions(+), 65 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index d126fe70dda0..2c0962ab9f29 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -496,7 +496,7 @@ export class ContainerRuntime extends EventEmitter containerScope, requestHandler); - // Create all internal stores if not already existing on storage or loaded a detached + // Create all internal data stores if not already existing on storage or loaded a detached // container from snapshot(ex. draft mode). if (!context.existing) { await runtime.createRootDataStore(schedulerId, schedulerId); @@ -1598,7 +1598,7 @@ export class ContainerRuntime extends EventEmitter const treeWithStats = convertToSummaryTree(snapshot, true); builder.addWithStats(key, treeWithStats); } else { - // If this component is not yet loaded, then there should be no changes in the snapshot from + // If this data store is not yet loaded, then there should be no changes in the snapshot from // which it was created as it is detached container. So just use the previous snapshot. assert(this.context.baseSnapshot, "BaseSnapshot should be there as detached container loaded from snapshot"); diff --git a/packages/runtime/datastore/src/localChannelContext.ts b/packages/runtime/datastore/src/localChannelContext.ts index 0c44e1b87a83..54a831912158 100644 --- a/packages/runtime/datastore/src/localChannelContext.ts +++ b/packages/runtime/datastore/src/localChannelContext.ts @@ -45,7 +45,7 @@ export class LocalChannelContext implements IChannelContext { registry: ISharedObjectRegistry, type: string, private readonly runtime: IFluidDataStoreRuntime, - private readonly componentContext: IFluidDataStoreContext, + private readonly dataStoreContext: IFluidDataStoreContext, private readonly storageService: IDocumentStorageService, private readonly submitFn: (content: any, localOpMetadata: unknown) => void, dirtyFn: (address: string) => void, @@ -74,7 +74,7 @@ export class LocalChannelContext implements IChannelContext { } public setConnectionState(connected: boolean, clientId?: string) { - // Connection events are ignored if the component is not yet attached + // Connection events are ignored if the data store is not yet attached if (!this.attached) { return; } @@ -172,7 +172,7 @@ export class LocalChannelContext implements IChannelContext { } this._services = createServiceEndpoints( this.id, - this.componentContext.connected, + this.dataStoreContext.connected, this.submitFn, this.dirtyFn, this.storageService, diff --git a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts index f6a0fae04093..6ac38ee1d1c8 100644 --- a/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/attachRegisterLocalApiTests.spec.ts @@ -5,11 +5,7 @@ import assert from "assert"; import { IRequest } from "@fluidframework/core-interfaces"; -import { - IFluidCodeDetails, - IProxyLoaderFactory, - AttachState, -} from "@fluidframework/container-definitions"; +import { IFluidCodeDetails, IProxyLoaderFactory, AttachState } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; @@ -31,7 +27,6 @@ describe(`Attach/Bind Api Tests For Attached Container`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const mapId1 = "mapId1"; const mapId2 = "mapId2"; diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 1ba8491fcd18..07c604371322 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -55,12 +55,12 @@ describe(`Dehydrate Rehydrate Container Test`, () => { async function createDetachedContainerAndGetRootComponent() { const container = await loader.createDetachedContainer(codeDetails); - // Get the root component from the detached container. + // Get the root data store from the detached container. const response = await container.request({ url: "/" }); - const defaultComponent = response.value; + const defaultDataStore = response.value; return { container, - defaultComponent, + defaultDataStore, }; } @@ -115,7 +115,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const snapshotTree = JSON.parse(container.serialize()); assert.strictEqual(Object.keys(snapshotTree.trees).length, 3, - "3 trees should be there(protocol, default component, scheduler"); + "3 trees should be there(protocol, default data store, scheduler"); assert.strictEqual(Object.keys(snapshotTree.trees[".protocol"].blobs).length, 8, "4 protocol blobs should be there(8 mappings)"); @@ -127,7 +127,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { assert.strictEqual(protocolAttributes.sequenceNumber, 0, "Seq number should be 0"); assert.strictEqual(protocolAttributes.minimumSequenceNumber, 0, "Min Seq number should be 0"); - // Check for default component + // Check for default data store const defaultComponentBlobId = snapshotTree.trees.default.blobs[".component"]; const componentAttributes = JSON.parse( Buffer.from(snapshotTree.trees.default.blobs[defaultComponentBlobId], "base64").toString()); @@ -135,18 +135,18 @@ describe(`Dehydrate Rehydrate Container Test`, () => { }); it("Dehydrated container snapshot 2 times with changes in between", async () => { - const { container, defaultComponent } = + const { container, defaultDataStore } = await createDetachedContainerAndGetRootComponent(); const snapshotTree1 = JSON.parse(container.serialize()); // Create a channel - const channel = defaultComponent.runtime.createChannel("test1", + const channel = defaultDataStore.runtime.createChannel("test1", "https://graph.microsoft.com/types/map") as SharedMap; channel.bindToContext(); const snapshotTree2 = JSON.parse(container.serialize()); assert.strictEqual(JSON.stringify(Object.keys(snapshotTree1.trees)), JSON.stringify(Object.keys(snapshotTree2.trees)), - "3 trees should be there(protocol, default component, scheduler"); + "3 trees should be there(protocol, default data store, scheduler"); // Check for protocol attributes const protocolAttributesBlobId1 = snapshotTree1.trees[".protocol"].blobs.attributes; @@ -167,23 +167,23 @@ describe(`Dehydrate Rehydrate Container Test`, () => { "Test channel 1 should be present in snapshot 2"); }); - it("Dehydrated container snapshot with component handle stored in map of other bound component", async () => { - const { container, defaultComponent } = + it("Dehydrated container snapshot with data store handle stored in map of other bound data store", async () => { + const { container, defaultDataStore } = await createDetachedContainerAndGetRootComponent(); - // Create another component - const peerComponent = await createPeerComponent(defaultComponent.context.containerRuntime); + // Create another data store + const peerComponent = await createPeerComponent(defaultDataStore.context.containerRuntime); const component2 = peerComponent.peerComponent as TestFluidComponent; // Create a channel const rootOfComponent1 = - await (defaultComponent as TestFluidComponent).getSharedObject(sharedMapId); + await (defaultDataStore as TestFluidComponent).getSharedObject(sharedMapId); rootOfComponent1.set("component2", component2.handle); const snapshotTree = JSON.parse(container.serialize()); assert.strictEqual(Object.keys(snapshotTree.trees).length, 4, "4 trees should be there"); - assert(snapshotTree.trees[component2.runtime.id], "Handle Bounded component should be in summary"); + assert(snapshotTree.trees[component2.runtime.id], "Handle Bounded data store should be in summary"); }); it("Rehydrate container from snapshot and check contents before attach", async () => { @@ -197,26 +197,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerComponent = schedulerResponse.value as ITestFluidComponent; - assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + const schedulerDataStore = schedulerResponse.value as ITestFluidComponent; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); - // Check for default component + // Check for default data store const response = await container2.request({ url: "/" }); assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultComponent = response.value as ITestFluidComponent; - assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + const defaultDataStore = response.value as ITestFluidComponent; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); // Check for dds - const sharedMap = await defaultComponent.getSharedObject(sharedMapId); - const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); - const sharedString = await defaultComponent.getSharedObject(sharedStringId); - const sharedCell = await defaultComponent.getSharedObject(sharedCellId); - const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); - const crc = await defaultComponent.getSharedObject>(crcId); - const coc = await defaultComponent.getSharedObject(cocId); - const ink = await defaultComponent.getSharedObject(sharedInkId); - const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); @@ -241,26 +241,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerComponent = schedulerResponse.value as ITestFluidComponent; - assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + const schedulerDataStore = schedulerResponse.value as ITestFluidComponent; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); - // Check for default component + // Check for default data store const response = await container2.request({ url: "/" }); assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultComponent = response.value as ITestFluidComponent; - assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + const defaultDataStore = response.value as ITestFluidComponent; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); // Check for dds - const sharedMap = await defaultComponent.getSharedObject(sharedMapId); - const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); - const sharedString = await defaultComponent.getSharedObject(sharedStringId); - const sharedCell = await defaultComponent.getSharedObject(sharedCellId); - const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); - const crc = await defaultComponent.getSharedObject>(crcId); - const coc = await defaultComponent.getSharedObject(cocId); - const ink = await defaultComponent.getSharedObject(sharedInkId); - const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); @@ -278,8 +278,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { await createDetachedContainerAndGetRootComponent(); const responseBefore = await container.request({ url: "/" }); - const defaultComponentBefore = responseBefore.value as ITestFluidComponent; - const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); + const defaultDataStoreBefore = responseBefore.value as ITestFluidComponent; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); sharedStringBefore.insertText(0, "Hello"); const snapshotTree = JSON.parse(container.serialize()); @@ -287,8 +287,8 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as ITestFluidComponent; - const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); + const defaultDataStoreAfter = responseAfter.value as ITestFluidComponent; + const sharedStringAfter = await defaultDataStoreAfter.getSharedObject(sharedStringId); assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), "Snapshot of shared string should match and contents should be same!!"); }); @@ -305,18 +305,18 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseBefore = await container2.request({ url: "/" }); - const defaultComponentBefore = responseBefore.value as ITestFluidComponent; - const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); - const sharedMapBefore = await defaultComponentBefore.getSharedObject(sharedMapId); + const defaultDataStoreBefore = responseBefore.value as ITestFluidComponent; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); + const sharedMapBefore = await defaultDataStoreBefore.getSharedObject(sharedMapId); str += "BB"; sharedStringBefore.insertText(0, str); sharedMapBefore.set("0", str); await container2.attach(request); const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as ITestFluidComponent; - const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); - const sharedMapAfter = await defaultComponentAfter.getSharedObject(sharedMapId); + const defaultDataStoreAfter = responseAfter.value as ITestFluidComponent; + const sharedStringAfter = await defaultDataStoreAfter.getSharedObject(sharedStringId); + const sharedMapAfter = await defaultDataStoreAfter.getSharedObject(sharedMapId); assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), "Snapshot of shared string should match and contents should be same!!"); assert.strictEqual(JSON.stringify(sharedMapAfter.snapshot()), JSON.stringify(sharedMapBefore.snapshot()), From 562b29b95f236b133ec7367a80ec55e1db2fd538 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 19 Aug 2020 16:29:43 -0700 Subject: [PATCH 23/27] merge conflict --- .../loader/container-loader/src/container.ts | 2 +- .../test/deRehydrateContainerTests.spec.ts | 168 +++++++++--------- 2 files changed, 82 insertions(+), 88 deletions(-) diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 0142119ea7dc..60affbe902cb 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -1026,7 +1026,7 @@ export class Container extends EventEmitterWithErrorHandling i // ...load in the existing quorum // Initialize the protocol handler - this.protocolHandler = + this._protocolHandler = await this.loadAndInitializeProtocolState(attributes, undefined, snapshotTree); await this.createDetachedContext(attributes, snapshotTree); diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 1ba8491fcd18..62123c8a7b20 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -4,32 +4,29 @@ */ import assert from "assert"; -import { - IFluidCodeDetails, - IProxyLoaderFactory, -} from "@fluidframework/container-definitions"; +import { IFluidCodeDetails, IProxyLoaderFactory } from "@fluidframework/container-definitions"; import { Loader } from "@fluidframework/container-loader"; import { IUrlResolver } from "@fluidframework/driver-definitions"; import { LocalDocumentServiceFactory, LocalResolver } from "@fluidframework/local-driver"; import { ILocalDeltaConnectionServer, LocalDeltaConnectionServer } from "@fluidframework/server-local-server"; import { LocalCodeLoader, - TestFluidComponentFactory, - ITestFluidComponent, - TestFluidComponent, + TestFluidObjectFactory, + ITestFluidObject, + TestFluidObject, } from "@fluidframework/test-utils"; import { SharedMap, SharedDirectory } from "@fluidframework/map"; import { IDocumentAttributes } from "@fluidframework/protocol-definitions"; import { IContainerRuntime } from "@fluidframework/container-runtime-definitions"; import { IContainerRuntimeBase } from "@fluidframework/runtime-definitions"; -import { IRequest } from "@fluidframework/core-interfaces"; -import { SharedString, SparseMatrix } from "@fluidframework/sequence"; import { ConsensusRegisterCollection } from "@fluidframework/register-collection"; -import { ConsensusQueue, ConsensusOrderedCollection } from "@fluidframework/ordered-collection"; -import { SharedMatrix } from "@fluidframework/matrix"; +import { SharedString, SparseMatrix } from "@fluidframework/sequence"; import { SharedCell } from "@fluidframework/cell"; import { Ink } from "@fluidframework/ink"; +import { SharedMatrix } from "@fluidframework/matrix"; +import { ConsensusQueue, ConsensusOrderedCollection } from "@fluidframework/ordered-collection"; import { SharedCounter } from "@fluidframework/counter"; +import { IRequest } from "@fluidframework/core-interfaces"; describe(`Dehydrate Rehydrate Container Test`, () => { const documentId = "deReHydrateContainerTest"; @@ -37,7 +34,6 @@ describe(`Dehydrate Rehydrate Container Test`, () => { package: "detachedContainerTestPackage1", config: {}, }; - const sharedStringId = "ss1Key"; const sharedMapId = "sm1Key"; const crcId = "crc1Key"; @@ -53,19 +49,19 @@ describe(`Dehydrate Rehydrate Container Test`, () => { let loader: Loader; let request: IRequest; - async function createDetachedContainerAndGetRootComponent() { + async function createDetachedContainerAndGetRootDataStore() { const container = await loader.createDetachedContainer(codeDetails); - // Get the root component from the detached container. + // Get the root dataStore from the detached container. const response = await container.request({ url: "/" }); - const defaultComponent = response.value; + const defaultDataStore = response.value; return { container, - defaultComponent, + defaultDataStore, }; } function createTestLoader(urlResolver: IUrlResolver): Loader { - const factory: TestFluidComponentFactory = new TestFluidComponentFactory([ + const factory: TestFluidObjectFactory = new TestFluidObjectFactory([ [sharedStringId, SharedString.getFactory()], [sharedMapId, SharedMap.getFactory()], [crcId, ConsensusRegisterCollection.getFactory()], @@ -77,7 +73,6 @@ describe(`Dehydrate Rehydrate Container Test`, () => { [sparseMatrixId, SparseMatrix.getFactory()], [sharedCounterId, SharedCounter.getFactory()], ]); - const codeLoader = new LocalCodeLoader([[codeDetails, factory]]); const documentServiceFactory = new LocalDocumentServiceFactory(testDeltaConnectionServer); return new Loader( @@ -89,16 +84,16 @@ describe(`Dehydrate Rehydrate Container Test`, () => { new Map()); } - const createPeerComponent = async ( + const createPeerDataStore = async ( containerRuntime: IContainerRuntimeBase, ) => { - const peerComponentRuntimeChannel = await (containerRuntime as IContainerRuntime) + const peerDataStoreRuntimeChannel = await (containerRuntime as IContainerRuntime) .createDataStoreWithRealizationFn(["default"]); - const peerComponent = - (await peerComponentRuntimeChannel.request({ url: "/" })).value as ITestFluidComponent; + const peerDataStore = + (await peerDataStoreRuntimeChannel.request({ url: "/" })).value as ITestFluidObject; return { - peerComponent, - peerComponentRuntimeChannel, + peerDataStore, + peerDataStoreRuntimeChannel, }; }; @@ -111,11 +106,11 @@ describe(`Dehydrate Rehydrate Container Test`, () => { it("Dehydrated container snapshot", async () => { const { container } = - await createDetachedContainerAndGetRootComponent(); + await createDetachedContainerAndGetRootDataStore(); const snapshotTree = JSON.parse(container.serialize()); assert.strictEqual(Object.keys(snapshotTree.trees).length, 3, - "3 trees should be there(protocol, default component, scheduler"); + "3 trees should be there(protocol, default dataStore, scheduler"); assert.strictEqual(Object.keys(snapshotTree.trees[".protocol"].blobs).length, 8, "4 protocol blobs should be there(8 mappings)"); @@ -127,26 +122,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { assert.strictEqual(protocolAttributes.sequenceNumber, 0, "Seq number should be 0"); assert.strictEqual(protocolAttributes.minimumSequenceNumber, 0, "Min Seq number should be 0"); - // Check for default component - const defaultComponentBlobId = snapshotTree.trees.default.blobs[".component"]; - const componentAttributes = JSON.parse( - Buffer.from(snapshotTree.trees.default.blobs[defaultComponentBlobId], "base64").toString()); - assert.strictEqual(componentAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); + // Check for default dataStore + const defaultDataStoreBlobId = snapshotTree.trees.default.blobs[".component"]; + const dataStoreAttributes = JSON.parse( + Buffer.from(snapshotTree.trees.default.blobs[defaultDataStoreBlobId], "base64").toString()); + assert.strictEqual(dataStoreAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); }); it("Dehydrated container snapshot 2 times with changes in between", async () => { - const { container, defaultComponent } = - await createDetachedContainerAndGetRootComponent(); + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); const snapshotTree1 = JSON.parse(container.serialize()); // Create a channel - const channel = defaultComponent.runtime.createChannel("test1", + const channel = defaultDataStore.runtime.createChannel("test1", "https://graph.microsoft.com/types/map") as SharedMap; channel.bindToContext(); const snapshotTree2 = JSON.parse(container.serialize()); assert.strictEqual(JSON.stringify(Object.keys(snapshotTree1.trees)), JSON.stringify(Object.keys(snapshotTree2.trees)), - "3 trees should be there(protocol, default component, scheduler"); + "3 trees should be there(protocol, default dataStore, scheduler"); // Check for protocol attributes const protocolAttributesBlobId1 = snapshotTree1.trees[".protocol"].blobs.attributes; @@ -167,28 +162,27 @@ describe(`Dehydrate Rehydrate Container Test`, () => { "Test channel 1 should be present in snapshot 2"); }); - it("Dehydrated container snapshot with component handle stored in map of other bound component", async () => { - const { container, defaultComponent } = - await createDetachedContainerAndGetRootComponent(); + it("Dehydrated container snapshot with dataStore handle stored in map of other bound dataStore", async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); - // Create another component - const peerComponent = await createPeerComponent(defaultComponent.context.containerRuntime); - const component2 = peerComponent.peerComponent as TestFluidComponent; + // Create another dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; // Create a channel - const rootOfComponent1 = - await (defaultComponent as TestFluidComponent).getSharedObject(sharedMapId); - rootOfComponent1.set("component2", component2.handle); + const rootOfDataStore1 = await (defaultDataStore as TestFluidObject).getSharedObject(sharedMapId); + rootOfDataStore1.set("dataStore2", dataStore2.handle); const snapshotTree = JSON.parse(container.serialize()); assert.strictEqual(Object.keys(snapshotTree.trees).length, 4, "4 trees should be there"); - assert(snapshotTree.trees[component2.runtime.id], "Handle Bounded component should be in summary"); + assert(snapshotTree.trees[dataStore2.runtime.id], "Handle Bounded dataStore should be in summary"); }); it("Rehydrate container from snapshot and check contents before attach", async () => { const { container } = - await createDetachedContainerAndGetRootComponent(); + await createDetachedContainerAndGetRootDataStore(); const snapshotTree = JSON.parse(container.serialize()); @@ -197,26 +191,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerComponent = schedulerResponse.value as ITestFluidComponent; - assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + const schedulerDataStore = schedulerResponse.value as TestFluidObject; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); - // Check for default component + // Check for default data store const response = await container2.request({ url: "/" }); assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultComponent = response.value as ITestFluidComponent; - assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + const defaultDataStore = response.value as TestFluidObject; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); // Check for dds - const sharedMap = await defaultComponent.getSharedObject(sharedMapId); - const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); - const sharedString = await defaultComponent.getSharedObject(sharedStringId); - const sharedCell = await defaultComponent.getSharedObject(sharedCellId); - const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); - const crc = await defaultComponent.getSharedObject>(crcId); - const coc = await defaultComponent.getSharedObject(cocId); - const ink = await defaultComponent.getSharedObject(sharedInkId); - const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); @@ -231,7 +225,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { it("Rehydrate container from snapshot and check contents after attach", async () => { const { container } = - await createDetachedContainerAndGetRootComponent(); + await createDetachedContainerAndGetRootDataStore(); const snapshotTree = JSON.parse(container.serialize()); @@ -241,26 +235,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerComponent = schedulerResponse.value as ITestFluidComponent; - assert.strictEqual(schedulerComponent.runtime.id, "_scheduler", "Id should be of scheduler"); + const schedulerDataStore = schedulerResponse.value as TestFluidObject; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); - // Check for default component + // Check for default data store const response = await container2.request({ url: "/" }); assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultComponent = response.value as ITestFluidComponent; - assert.strictEqual(defaultComponent.runtime.id, "default", "Id should be default"); + const defaultDataStore = response.value as TestFluidObject; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); // Check for dds - const sharedMap = await defaultComponent.getSharedObject(sharedMapId); - const sharedDir = await defaultComponent.getSharedObject(sharedDirectoryId); - const sharedString = await defaultComponent.getSharedObject(sharedStringId); - const sharedCell = await defaultComponent.getSharedObject(sharedCellId); - const sharedCounter = await defaultComponent.getSharedObject(sharedCounterId); - const crc = await defaultComponent.getSharedObject>(crcId); - const coc = await defaultComponent.getSharedObject(cocId); - const ink = await defaultComponent.getSharedObject(sharedInkId); - const sharedMatrix = await defaultComponent.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultComponent.getSharedObject(sparseMatrixId); + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); @@ -275,11 +269,11 @@ describe(`Dehydrate Rehydrate Container Test`, () => { it("Change contents of dds, then rehydrate and then check snapshot", async () => { const { container } = - await createDetachedContainerAndGetRootComponent(); + await createDetachedContainerAndGetRootDataStore(); const responseBefore = await container.request({ url: "/" }); - const defaultComponentBefore = responseBefore.value as ITestFluidComponent; - const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); + const defaultDataStoreBefore = responseBefore.value as TestFluidObject; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); sharedStringBefore.insertText(0, "Hello"); const snapshotTree = JSON.parse(container.serialize()); @@ -287,7 +281,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as ITestFluidComponent; + const defaultComponentAfter = responseAfter.value as TestFluidObject; const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), "Snapshot of shared string should match and contents should be same!!"); @@ -295,26 +289,26 @@ describe(`Dehydrate Rehydrate Container Test`, () => { it("Rehydrate container from snapshot, change contents of dds and then check snapshot", async () => { const { container } = - await createDetachedContainerAndGetRootComponent(); + await createDetachedContainerAndGetRootDataStore(); let str = "AA"; const response1 = await container.request({ url: "/" }); - const defaultComponent1 = response1.value as ITestFluidComponent; + const defaultComponent1 = response1.value as TestFluidObject; const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); sharedString1.insertText(0, str); const snapshotTree = JSON.parse(container.serialize()); const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); const responseBefore = await container2.request({ url: "/" }); - const defaultComponentBefore = responseBefore.value as ITestFluidComponent; - const sharedStringBefore = await defaultComponentBefore.getSharedObject(sharedStringId); - const sharedMapBefore = await defaultComponentBefore.getSharedObject(sharedMapId); + const defaultDataStoreBefore = responseBefore.value as TestFluidObject; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); + const sharedMapBefore = await defaultDataStoreBefore.getSharedObject(sharedMapId); str += "BB"; sharedStringBefore.insertText(0, str); sharedMapBefore.set("0", str); await container2.attach(request); const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as ITestFluidComponent; + const defaultComponentAfter = responseAfter.value as TestFluidObject; const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); const sharedMapAfter = await defaultComponentAfter.getSharedObject(sharedMapId); assert.strictEqual(JSON.stringify(sharedStringAfter.snapshot()), JSON.stringify(sharedStringBefore.snapshot()), From 55d76e298c274db293d6e3d733e1ffffc5311721 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 19 Aug 2020 21:44:12 -0700 Subject: [PATCH 24/27] cast --- packages/runtime/datastore/src/dataStoreRuntime.ts | 3 ++- .../runtime/runtime-definitions/src/dataStoreContext.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index a9f53a079ab2..c328f88c29bd 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -49,6 +49,7 @@ import { IInboundSignalMessage, ISummaryTreeWithStats, CreateSummarizerNodeSource, + IFluidDataStoreContextType, } from "@fluidframework/runtime-definitions"; import { generateHandleContextPath, SummaryTreeBuilder } from "@fluidframework/runtime-utils"; import { @@ -204,7 +205,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto Object.keys(tree.trees).forEach((path) => { let channelContext: IChannelContext; // If already exists on storage, then create a remote channel. - if (dataStoreContext.isLocalDataStore) { + if ((dataStoreContext as IFluidDataStoreContextType).isLocalDataStore) { const channelAttributes = readAndParseFromBlobs( tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); channelContext = new LocalChannelContext( diff --git a/packages/runtime/runtime-definitions/src/dataStoreContext.ts b/packages/runtime/runtime-definitions/src/dataStoreContext.ts index 91c000258eff..8a4bf05088a3 100644 --- a/packages/runtime/runtime-definitions/src/dataStoreContext.ts +++ b/packages/runtime/runtime-definitions/src/dataStoreContext.ts @@ -254,8 +254,7 @@ export interface IFluidDataStoreContext extends EventEmitter { readonly branch: string; readonly baseSnapshot: ISnapshotTree | undefined; readonly loader: ILoader; - // True if the data store is LocalDataStoreContext. - readonly isLocalDataStore: boolean; + /** * Indicates the attachment state of the data store to a host service. */ @@ -355,3 +354,8 @@ export interface IFluidDataStoreContext extends EventEmitter { */ composeSubpackagePath(subpackage: string): Promise; } + +export interface IFluidDataStoreContextType extends IFluidDataStoreContext { + // True if the data store is LocalDataStoreContext. + readonly isLocalDataStore: boolean; +} From d18fd5fa4f39b8b56fe403619db0759946f940ad Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Wed, 19 Aug 2020 21:45:21 -0700 Subject: [PATCH 25/27] cast --- packages/runtime/runtime-definitions/src/dataStoreContext.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/runtime/runtime-definitions/src/dataStoreContext.ts b/packages/runtime/runtime-definitions/src/dataStoreContext.ts index 8a4bf05088a3..f8f2fbaac058 100644 --- a/packages/runtime/runtime-definitions/src/dataStoreContext.ts +++ b/packages/runtime/runtime-definitions/src/dataStoreContext.ts @@ -254,7 +254,6 @@ export interface IFluidDataStoreContext extends EventEmitter { readonly branch: string; readonly baseSnapshot: ISnapshotTree | undefined; readonly loader: ILoader; - /** * Indicates the attachment state of the data store to a host service. */ From c39577307d258f50924a2082c182059c05f798f0 Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Fri, 28 Aug 2020 11:40:46 -0700 Subject: [PATCH 26/27] pr sugg --- .../container-definitions/src/loader.ts | 4 ++-- .../loader/container-loader/src/container.ts | 20 ++++++++++++------- .../loader/container-loader/src/loader.ts | 18 ++++++++++------- .../loader/driver-utils/src/readAndParse.ts | 4 +--- .../src/webWorkerLoader.ts | 4 ++-- packages/runtime/datastore/src/utils.ts | 4 ++-- .../test/deRehydrateContainerTests.spec.ts | 8 ++++---- 7 files changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/loader/container-definitions/src/loader.ts b/packages/loader/container-definitions/src/loader.ts index ef8903c753ad..98e22e779a5e 100644 --- a/packages/loader/container-definitions/src/loader.ts +++ b/packages/loader/container-definitions/src/loader.ts @@ -149,13 +149,13 @@ export interface ILoader { * Creates a new container using the specified chaincode but in an unattached state. While unattached all * updates will only be local until the user explicitly attaches the container to a service provider. */ - createDetachedContainer(source: IFluidCodeDetails): Promise; + createDetachedContainer(codeDetails: IFluidCodeDetails): Promise; /** * Creates a new container using the specified snapshot but in an unattached state. While unattached all * updates will only be local until the user explicitly attaches the container to a service provider. */ - createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise; + rehydrateDetachedContainerFromSnapshot(snapshot: ISnapshotTree): Promise; } export enum LoaderHeader { diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index ed9f49082e54..bb0e9ad61922 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -124,6 +124,14 @@ export enum ConnectionState { Connected, } +export type DetachedContainerSource = { + codeDetails: IFluidCodeDetails, + create: true, +} | { + snapshot: ISnapshotTree, + create: false, +}; + export class Container extends EventEmitterWithErrorHandling implements IContainer { public static version = "^0.1.0"; @@ -194,8 +202,7 @@ export class Container extends EventEmitterWithErrorHandling i options: any, scope: IFluidObject, loader: Loader, - codeDetails: IFluidCodeDetails | undefined, - snapshot: ISnapshotTree | undefined, + source: DetachedContainerSource, serviceFactory: IDocumentServiceFactory, urlResolver: IUrlResolver, logger?: ITelemetryBaseLogger, @@ -210,11 +217,10 @@ export class Container extends EventEmitterWithErrorHandling i {}, logger); - if (snapshot !== undefined) { - await container.createDetachedFromSnapshot(snapshot); + if (source.create) { + await container.createDetached(source.codeDetails); } else { - assert(codeDetails, "One of the source should be there to load from!!"); - await container.createDetached(codeDetails); + await container.rehydrateDetachedFromSnapshot(source.snapshot); } return container; @@ -1047,7 +1053,7 @@ export class Container extends EventEmitterWithErrorHandling i this.propagateConnectionState(); } - private async createDetachedFromSnapshot(snapshotTree: ISnapshotTree) { + private async rehydrateDetachedFromSnapshot(snapshotTree: ISnapshotTree) { const attributes = await this.getDocumentAttributes(undefined, snapshotTree); assert.strictEqual(attributes.sequenceNumber, 0, "Seq number in detached container should be 0!!"); this.attachDeltaManagerOpHandler(attributes); diff --git a/packages/loader/container-loader/src/loader.ts b/packages/loader/container-loader/src/loader.ts index 4bf53bf0a191..b6d2ecfeb33c 100644 --- a/packages/loader/container-loader/src/loader.ts +++ b/packages/loader/container-loader/src/loader.ts @@ -103,7 +103,7 @@ export class RelativeLoader extends EventEmitter implements ILoader { throw new Error("Relative loader should not create a detached container"); } - public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { + public async rehydrateDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { throw new Error("Relative loader should not create a detached container from snapshot"); } @@ -160,7 +160,7 @@ export class Loader extends EventEmitter implements ILoader { this.documentServiceFactory = MultiDocumentServiceFactory.create(documentServiceFactory); } - public async createDetachedContainer(source: IFluidCodeDetails): Promise { + public async createDetachedContainer(codeDetails: IFluidCodeDetails): Promise { debug(`Container creating in detached state: ${performanceNow()} `); return Container.create( @@ -168,14 +168,16 @@ export class Loader extends EventEmitter implements ILoader { this.options, this.scope, this, - source, - undefined, + { + codeDetails, + create: true, + }, this.documentServiceFactory, this.resolver, this.subLogger); } - public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { + public async rehydrateDetachedContainerFromSnapshot(snapshot: ISnapshotTree): Promise { debug(`Container creating in detached state: ${performanceNow()} `); return Container.create( @@ -183,8 +185,10 @@ export class Loader extends EventEmitter implements ILoader { this.options, this.scope, this, - undefined, - source, + { + snapshot, + create: false, + }, this.documentServiceFactory, this.resolver, this.subLogger); diff --git a/packages/loader/driver-utils/src/readAndParse.ts b/packages/loader/driver-utils/src/readAndParse.ts index 35368038f9ca..be82c096a4d1 100644 --- a/packages/loader/driver-utils/src/readAndParse.ts +++ b/packages/loader/driver-utils/src/readAndParse.ts @@ -28,8 +28,6 @@ export async function readAndParse(storage: Pick(blobs: {[index: string]: string}, id: string): T { const encoded = blobs[id]; - const decoded = Buffer - .from(encoded, "base64") - .toString(); + const decoded = fromBase64ToUtf8(encoded); return JSON.parse(decoded) as T; } diff --git a/packages/loader/execution-context-loader/src/webWorkerLoader.ts b/packages/loader/execution-context-loader/src/webWorkerLoader.ts index 3a7bfaa74471..cf310501f91e 100644 --- a/packages/loader/execution-context-loader/src/webWorkerLoader.ts +++ b/packages/loader/execution-context-loader/src/webWorkerLoader.ts @@ -76,7 +76,7 @@ export class WebWorkerLoader implements ILoader, IFluidRunnable, IFluidRouter { return this.proxy.createDetachedContainer(source); } - public async createDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { - return this.proxy.createDetachedContainerFromSnapshot(source); + public async rehydrateDetachedContainerFromSnapshot(source: ISnapshotTree): Promise { + return this.proxy.rehydrateDetachedContainerFromSnapshot(source); } } diff --git a/packages/runtime/datastore/src/utils.ts b/packages/runtime/datastore/src/utils.ts index 4ca462965d7c..9ddbc49df01c 100644 --- a/packages/runtime/datastore/src/utils.ts +++ b/packages/runtime/datastore/src/utils.ts @@ -3,9 +3,9 @@ * Licensed under the MIT License. */ -import { Buffer } from "buffer"; import { ISnapshotTree, ITreeEntry, ITree } from "@fluidframework/protocol-definitions"; import { BlobTreeEntry, TreeTreeEntry } from "@fluidframework/protocol-base"; +import { fromBase64ToUtf8 } from "@fluidframework/common-utils"; export function convertSnapshotToITree(snapshotTree: ISnapshotTree): ITree { const entries: ITreeEntry[] = []; @@ -17,7 +17,7 @@ export function convertSnapshotToITree(snapshotTree: ISnapshotTree): ITree { } } for (const [key, value] of Object.entries(blobMapFinal)) { - const decoded = Buffer.from(value, "base64").toString(); + const decoded = fromBase64ToUtf8(value); entries.push(new BlobTreeEntry(key, decoded)); } for (const [key, tree] of Object.entries(snapshotTree.trees)) { diff --git a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index f3b29bd7e05b..29f3243183c7 100644 --- a/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -184,7 +184,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const snapshotTree = JSON.parse(container.serialize()); - const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); // Check for scheduler const schedulerResponse = await container2.request({ url: "_scheduler" }); @@ -227,7 +227,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const snapshotTree = JSON.parse(container.serialize()); - const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); await container2.attach(request); // Check for scheduler @@ -276,7 +276,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const snapshotTree = JSON.parse(container.serialize()); - const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); const responseAfter = await container2.request({ url: "/" }); const defaultComponentAfter = responseAfter.value as TestFluidObject; @@ -295,7 +295,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { sharedString1.insertText(0, str); const snapshotTree = JSON.parse(container.serialize()); - const container2 = await loader.createDetachedContainerFromSnapshot(snapshotTree); + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); const responseBefore = await container2.request({ url: "/" }); const defaultDataStoreBefore = responseBefore.value as TestFluidObject; const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); From a1a07a8932f38042ccd7cbdb4e4f6a756126b7dc Mon Sep 17 00:00:00 2001 From: Jatin Garg Date: Tue, 8 Sep 2020 12:04:55 -0700 Subject: [PATCH 27/27] pr sugg --- common/lib/common-utils/src/index.ts | 1 + .../runtime/container-runtime/src/containerRuntime.ts | 8 ++++---- .../runtime/container-runtime/src/dataStoreContext.ts | 8 ++++++-- packages/runtime/datastore/src/dataStoreRuntime.ts | 6 +++--- packages/runtime/datastore/src/localChannelContext.ts | 5 +++++ 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/common/lib/common-utils/src/index.ts b/common/lib/common-utils/src/index.ts index 8807f074be89..f01f018b2c2e 100644 --- a/common/lib/common-utils/src/index.ts +++ b/common/lib/common-utils/src/index.ts @@ -19,3 +19,4 @@ export * from "./timer"; export * from "./trace"; export * from "./typedEventEmitter"; export * from "./unreachable"; +export * from "./lazy"; diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 2766063ce292..194dba7089b2 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -751,8 +751,7 @@ export class ContainerRuntime extends EventEmitter // Extract stores stored inside the snapshot const fluidDataStores = new Map(); - // back-compat 0.24 baseSnapshotCouldBeNull - if (context.baseSnapshot !== undefined && context.baseSnapshot !== null) { + if (typeof context.baseSnapshot === "object") { const baseSnapshot = context.baseSnapshot; Object.keys(baseSnapshot.trees).forEach((value) => { if (value !== ".protocol" && value !== ".logTail" && value !== ".serviceProtocol") { @@ -777,8 +776,7 @@ export class ContainerRuntime extends EventEmitter this.summarizerNode.getCreateChildFn(key, { type: CreateSummarizerNodeSource.FromSummary })); } else { let pkgFromSnapshot: string[]; - // back-compat 0.24 baseSnapshotCouldBeNull - if (context.baseSnapshot === null || context.baseSnapshot === undefined) { + if (typeof context.baseSnapshot !== "object") { throw new Error("Snapshot should be there to load from!!"); } const snapshotTree = value as ISnapshotTree; @@ -788,6 +786,8 @@ export class ContainerRuntime extends EventEmitter snapshotTree.blobs[".component"]); // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. // For snapshotFormatVersion = "0.1", pkg is jsonified, otherwise it is just a string. + // However the feature of loading a detached container from snapshot, is added when the + // snapshotFormatVersion is "0.1", so we don't expect it to be anything else. if (snapshotFormatVersion === currentSnapshotFormatVersion) { pkgFromSnapshot = JSON.parse(pkg) as string[]; } else { diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index db3e2780224b..d1d8b7a0ec82 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -758,6 +758,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { summaryTracker: SummaryTracker, createSummarizerNode: CreateChildSummarizerNodeFn, bindChannel: (channel: IFluidDataStoreChannel) => void, + snapshotTree: ISnapshotTree | undefined, ) { super( id, @@ -767,7 +768,8 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { scope, summaryTracker, createSummarizerNode, - bindChannel); + bindChannel, + snapshotTree); } } @@ -789,6 +791,7 @@ export class LocalDetachedFluidDataStoreContext summaryTracker: SummaryTracker, createSummarizerNode: CreateChildSummarizerNodeFn, bindChannel: (channel: IFluidDataStoreChannel) => void, + snapshotTree: ISnapshotTree | undefined, ) { super( id, @@ -798,7 +801,8 @@ export class LocalDetachedFluidDataStoreContext scope, summaryTracker, createSummarizerNode, - bindChannel); + bindChannel, + snapshotTree); assert(this.pkg === undefined); this.detachedRuntimeCreation = true; } diff --git a/packages/runtime/datastore/src/dataStoreRuntime.ts b/packages/runtime/datastore/src/dataStoreRuntime.ts index e399eb3a822c..24918b1a0ad9 100644 --- a/packages/runtime/datastore/src/dataStoreRuntime.ts +++ b/packages/runtime/datastore/src/dataStoreRuntime.ts @@ -93,7 +93,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto ): FluidDataStoreRuntime { const logger = ChildLogger.create(context.containerRuntime.logger, undefined, { dataStoreId: uuid() }); const runtime = new FluidDataStoreRuntime( - context, + context as IFluidDataStoreContextType, context.documentId, context.id, context.parentBranch, @@ -176,7 +176,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto private _attachState: AttachState; private constructor( - private readonly dataStoreContext: IFluidDataStoreContext, + private readonly dataStoreContext: IFluidDataStoreContextType, public readonly documentId: string, public readonly id: string, public readonly parentBranch: string | null, @@ -199,7 +199,7 @@ export class FluidDataStoreRuntime extends EventEmitter implements IFluidDataSto Object.keys(tree.trees).forEach((path) => { let channelContext: IChannelContext; // If already exists on storage, then create a remote channel. - if ((dataStoreContext as IFluidDataStoreContextType).isLocalDataStore) { + if (dataStoreContext.isLocalDataStore) { const channelAttributes = readAndParseFromBlobs( tree.trees[path].blobs, tree.trees[path].blobs[".attributes"]); channelContext = new LocalChannelContext( diff --git a/packages/runtime/datastore/src/localChannelContext.ts b/packages/runtime/datastore/src/localChannelContext.ts index c010c8f2a350..d02e38dc4d94 100644 --- a/packages/runtime/datastore/src/localChannelContext.ts +++ b/packages/runtime/datastore/src/localChannelContext.ts @@ -84,9 +84,14 @@ export class LocalChannelContext implements IChannelContext { public processOp(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown): void { assert(this.attached, "Local channel must be attached when processing op"); + // A local channel may not be loaded in case where we rehydrate the container from a snapshot because of + // delay loading. So after the container is attached and some other client joins which start generating + // ops for this channel. So not loaded local channel can still receive ops and we store them to process later. if (this.isLoaded) { this.services.deltaConnection.process(message, local, localOpMetadata); } else { + assert.strictEqual(local, false, + "Should always be remote because a local dds shouldn't generate ops before loading"); this.pending.push(message); } }