Remove convertToSummaryTreeWithStats from DDSs

BREAKING.md

@@ -27,7 +27,8 @@ The `getQuorum()` method on `IContainer` and the `quorum` member of `IContainerC
 
 ### `SharedObject` summary and GC API changes
 
-`SharedObject.snapshotCore` is renamed to `summarizeCore` and returns `ISummaryTreeWithStats`. A temporary way to fix this up quickly is to call `convertToSummaryTreeWithStats` on the `ITree` previously returned, but `convertToSummaryTreeWithStats` will be deprecated in the future and `ISummaryTreeWithStats` should be created directly.
+`SharedObject.snapshotCore` is renamed to `summarizeCore` and returns `ISummaryTreeWithStats`. Use
+`SummaryTreeBuilder` to create a summary instead of `ITree`.
 
 `SharedObject.getGCDataCore` is renamed to `processGCDataCore` and a `SummarySerializer` is passed as a parameter. The method should run the serializer over the handles as before and does not need to return anything. The caller will extract the GC data from the serializer.
 
@@ -102,11 +103,11 @@ As-written above, these promises will silently remain pending forever if the key
 
 ### Remove Legacy Data Object and Factories
 
-In order to ease migration to the new Aqueduct Data Object and Data Object Factory generic arguments we added legacy versions of those classes in version 0.53. 
+In order to ease migration to the new Aqueduct Data Object and Data Object Factory generic arguments we added legacy versions of those classes in version 0.53.
 
 In this release we remove those legacy classes: LegacyDataObject, LegacyPureDataObject, LegacyDataObjectFactory, and LegacyPureDataObjectFactory
 
-It is recommend you migrate to the new generic arguments before consuming this release. 
+It is recommend you migrate to the new generic arguments before consuming this release.
 Details are here: [0.53: Generic Argument Changes to DataObjects and Factories](#Generic-Argument-Changes-to-DataObjects-and-Factories)
 
 ### Removed `innerRequestHandler`

api-report/merge-tree.api.md

@@ -9,8 +9,8 @@ import { IFluidDataStoreRuntime } from '@fluidframework/datastore-definitions';
 import { IFluidHandle } from '@fluidframework/core-interfaces';
 import { IFluidSerializer } from '@fluidframework/shared-object-base';
 import { ISequencedDocumentMessage } from '@fluidframework/protocol-definitions';
+import { ISummaryTreeWithStats } from '@fluidframework/runtime-definitions';
 import { ITelemetryLogger } from '@fluidframework/common-definitions';
-import { ITree } from '@fluidframework/protocol-definitions';
 import { Trace } from '@fluidframework/common-utils';
 
 // @public (undocumented)
@@ -190,12 +190,12 @@ export class Client {
     resolveRemoteClientPosition(remoteClientPosition: number, remoteClientRefSeq: number, remoteClientId: string): number | undefined;
     serializeGCData(handle: IFluidHandle, handleCollectingSerializer: IFluidSerializer): void;
     // (undocumented)
-    snapshot(runtime: IFluidDataStoreRuntime, handle: IFluidHandle, serializer: IFluidSerializer, catchUpMsgs: ISequencedDocumentMessage[]): ITree;
-    // (undocumented)
     readonly specToSegment: (spec: IJSONSegment) => ISegment;
     // (undocumented)
     startOrUpdateCollaboration(longClientId: string | undefined, minSeq?: number, currentSeq?: number): void;
     // (undocumented)
+    summarize(runtime: IFluidDataStoreRuntime, handle: IFluidHandle, serializer: IFluidSerializer, catchUpMsgs: ISequencedDocumentMessage[]): ISummaryTreeWithStats;
+    // (undocumented)
     updateConsensusProperty(op: IMergeTreeAnnotateMsg, msg: ISequencedDocumentMessage): void;
     // (undocumented)
     updateMinSeq(minSeq: number): void;
@@ -1448,7 +1448,7 @@ export class SnapshotLegacy {
     constructor(mergeTree: MergeTree, logger: ITelemetryLogger, filename?: string | undefined, onCompletion?: (() => void) | undefined);
     // (undocumented)
     static readonly body = "body";
-    emit(catchUpMsgs: ISequencedDocumentMessage[], serializer: IFluidSerializer, bind: IFluidHandle): ITree;
+    emit(catchUpMsgs: ISequencedDocumentMessage[], serializer: IFluidSerializer, bind: IFluidHandle): ISummaryTreeWithStats;
     // (undocumented)
     extractSync(): IJSONSegment[];
     // (undocumented)

experimental/PropertyDDS/packages/property-dds/src/propertyTree.ts

@@ -12,7 +12,6 @@ import {copy as cloneDeep} from "fastest-json-copy";
 import { AttachState } from "@fluidframework/container-definitions";
 import {
 	ISequencedDocumentMessage,
-	ITree,
 	MessageType,
 	FileMode,
 	TreeEntry,
@@ -27,8 +26,8 @@ import {
 
 import { bufferToString, assert } from "@fluidframework/common-utils";
 import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { convertToSummaryTreeWithStats } from "@fluidframework/runtime-utils";
 import { IFluidSerializer, SharedObject } from "@fluidframework/shared-object-base";
+import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 
 import {
 	ChangeSet,
@@ -399,7 +398,7 @@ export class SharedPropertyTree extends SharedObject {
 			numChunks: 0,
 		};
 
-		const chunks: ITreeEntry[] = [];
+		const builder = new SummaryTreeBuilder();
 		if (!this.useMH) {
 			// If the MH is not used, we have to include the tip view, the remote changes and the received
 			// deltas to the summary
@@ -416,7 +415,7 @@ export class SharedPropertyTree extends SharedObject {
 			// the chunking code below could create chunks which are bigger than the
 			// allowed limit after encoding the JSON via UTF8 encoding. To make sure
 			// the encoded string stays within the size limit, we replace unicode characters
-			// with the cooresponding escapes. This way, it won't change size when encoded as
+			// with the corresponding escapes. This way, it won't change size when encoded as
 			// utf8
 			serializedSummary = serializedSummary.replace(
 				/[\u007F-\uFFFF]/g,
@@ -425,37 +424,15 @@ export class SharedPropertyTree extends SharedObject {
 			);
 
 			for (let pos = 0, i = 0; pos < serializedSummary.length; pos += chunkSize, i++) {
-				chunks.push({
-					path: `summaryChunk_${i}`,
-					mode: FileMode.File,
-					type: TreeEntry.Blob,
-					value: {
-						contents: serializedSummary.substr(pos, chunkSize),
-						encoding: "utf-8",
-					},
-				});
+				builder.addBlob(`summaryChunk_${i}`, serializedSummary.substr(pos, chunkSize));
+				snapshot.numChunks++;
 			}
-			snapshot.numChunks = chunks.length;
 		}
 
-		return convertToSummaryTreeWithStats({
-			entries: [
-				{
-					path: "properties",
-					mode: FileMode.File,
-					type: TreeEntry.Blob,
-					value: {
-						contents:
-							serializer !== undefined
-								? serializer.stringify(snapshot, this.handle)
-								: JSON.stringify(snapshot),
-						encoding: "utf-8",
-					},
-				},
-				...chunks,
-			],
-			id: undefined,
-		}, fullTree);
+		builder.addBlob("properties", serializer !== undefined
+			? serializer.stringify(snapshot, this.handle)
+			: JSON.stringify(snapshot));
+		return builder.getSummaryTree();
 	}
 
 	protected async loadCore(storage: IChannelStorageService): Promise<void> {

packages/dds/map/package.json

@@ -60,7 +60,6 @@
     "@fluidframework/core-interfaces": "^0.41.0",
     "@fluidframework/datastore-definitions": "^0.55.0",
     "@fluidframework/driver-utils": "^0.55.0",
-    "@fluidframework/protocol-base": "^0.1034.0",
     "@fluidframework/protocol-definitions": "^0.1026.0",
     "@fluidframework/runtime-definitions": "^0.55.0",
     "@fluidframework/runtime-utils": "^0.55.0",

packages/dds/map/src/directory.ts

@@ -5,11 +5,8 @@
 
 import { assert,TypedEventEmitter } from "@fluidframework/common-utils";
 import { readAndParse } from "@fluidframework/driver-utils";
-import { addBlobToTree } from "@fluidframework/protocol-base";
-import { convertToSummaryTreeWithStats } from "@fluidframework/runtime-utils";
 import {
     ISequencedDocumentMessage,
-    ITree,
     MessageType,
 } from "@fluidframework/protocol-definitions";
 import {
@@ -21,6 +18,7 @@ import {
 } from "@fluidframework/datastore-definitions";
 import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { IFluidSerializer, SharedObject, ValueType } from "@fluidframework/shared-object-base";
+import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import * as path from "path-browserify";
 import {
     IDirectory,
@@ -205,10 +203,10 @@ export interface IDirectoryNewStorageFormat {
     content: IDirectoryDataObject;
 }
 
-function serializeDirectory(root: SubDirectory, serializer: IFluidSerializer): ITree {
+function serializeDirectory(root: SubDirectory, serializer: IFluidSerializer): ISummaryTreeWithStats {
     const MinValueSizeSeparateSnapshotBlob = 8 * 1024;
 
-    const tree: ITree = { entries: [] };
+    const builder = new SummaryTreeBuilder();
     let counter = 0;
     const blobs: string[] = [];
 
@@ -242,7 +240,7 @@ function serializeDirectory(root: SubDirectory, serializer: IFluidSerializer): I
                 const blobName = `blob${counter}`;
                 counter++;
                 blobs.push(blobName);
-                addBlobToTree(tree, blobName, extraContent);
+                builder.addBlob(blobName, JSON.stringify(extraContent));
             } else {
                 currentSubDirObject.storage[key] = result;
             }
@@ -262,9 +260,9 @@ function serializeDirectory(root: SubDirectory, serializer: IFluidSerializer): I
         blobs,
         content,
     };
-    addBlobToTree(tree, snapshotFileName, newFormat);
+    builder.addBlob(snapshotFileName, JSON.stringify(newFormat));
 
-    return tree;
+    return builder.getSummaryTree();
 }
 
 /**
@@ -565,8 +563,7 @@ export class SharedDirectory extends SharedObject<ISharedDirectoryEvents> implem
      * @internal
      */
     protected summarizeCore(serializer: IFluidSerializer, fullTree: boolean): ISummaryTreeWithStats {
-        const snapshot = serializeDirectory(this.root, serializer);
-        return convertToSummaryTreeWithStats(snapshot, fullTree);
+        return serializeDirectory(this.root, serializer);
     }
 
     /**

packages/dds/matrix/src/handletable.ts

@@ -77,7 +77,7 @@ export class HandleTable<T> {
     private get next() { return this.handles[0] as Handle; }
     private set next(handle: Handle) { this.handles[0] = handle; }
 
-    public snapshot() {
+    public getSummaryContent() {
         return this.handles;
     }
 

packages/dds/matrix/src/matrix.ts

@@ -4,12 +4,7 @@
  */
 
 import { assert } from "@fluidframework/common-utils";
-import {
-    FileMode,
-    ISequencedDocumentMessage,
-    ITree,
-    TreeEntry,
-} from "@fluidframework/protocol-definitions";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import {
     IFluidDataStoreRuntime,
     IChannelStorageService,
@@ -24,7 +19,7 @@ import {
     SummarySerializer,
 } from "@fluidframework/shared-object-base";
 import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { convertToSummaryTreeWithStats, ObjectStoragePartition } from "@fluidframework/runtime-utils";
+import { ObjectStoragePartition, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import {
     IMatrixProducer,
     IMatrixConsumer,
@@ -37,7 +32,7 @@ import { PermutationVector, PermutationSegment } from "./permutationvector";
 import { SparseArray2D } from "./sparsearray2d";
 import { SharedMatrixFactory } from "./runtime";
 import { Handle, isHandleValid } from "./handletable";
-import { deserializeBlob, serializeBlob } from "./serialization";
+import { deserializeBlob } from "./serialization";
 import { ensureRange } from "./range";
 import { IUndoConsumer } from "./types";
 import { MatrixUndoProvider } from "./undoprovider";
@@ -430,32 +425,15 @@ export class SharedMatrix<T = any>
     }
 
     protected summarizeCore(serializer: IFluidSerializer, fullTree: boolean): ISummaryTreeWithStats {
-        const tree: ITree = {
-            entries: [
-                {
-                    mode: FileMode.Directory,
-                    path: SnapshotPath.rows,
-                    type: TreeEntry.Tree,
-                    value: this.rows.snapshot(this.runtime, this.handle, serializer),
-                },
-                {
-                    mode: FileMode.Directory,
-                    path: SnapshotPath.cols,
-                    type: TreeEntry.Tree,
-                    value: this.cols.snapshot(this.runtime, this.handle, serializer),
-                },
-                serializeBlob(
-                    this.handle,
-                    SnapshotPath.cells,
-                    [
-                        this.cells.snapshot(),
-                        this.pending.snapshot(),
-                    ],
-                    serializer),
-            ],
-        };
-
-        return convertToSummaryTreeWithStats(tree, fullTree);
+        const builder = new SummaryTreeBuilder();
+        builder.addWithStats(SnapshotPath.rows, this.rows.summarize(this.runtime, this.handle, serializer));
+        builder.addWithStats(SnapshotPath.cols, this.cols.summarize(this.runtime, this.handle, serializer));
+        builder.addBlob(SnapshotPath.cells,
+            serializer.stringify([
+                this.cells.snapshot(),
+                this.pending.snapshot(),
+            ], this.handle));
+        return builder.getSummaryTree();
     }
 
     /**

packages/dds/matrix/src/permutationvector.ts

@@ -21,11 +21,11 @@ import {
     ReferenceType,
 } from "@fluidframework/merge-tree";
 import { IFluidHandle } from "@fluidframework/core-interfaces";
-import { FileMode, TreeEntry, ITree } from "@fluidframework/protocol-definitions";
 import { IFluidSerializer } from "@fluidframework/shared-object-base";
-import { ObjectStoragePartition } from "@fluidframework/runtime-utils";
+import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
+import { ObjectStoragePartition, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import { HandleTable, Handle, isHandleValid } from "./handletable";
-import { serializeBlob, deserializeBlob } from "./serialization";
+import { deserializeBlob } from "./serialization";
 import { HandleCache } from "./handlecache";
 import { VectorUndoProvider } from "./undoprovider";
 
@@ -286,19 +286,13 @@ export class PermutationVector extends Client {
         return this.findReconnectionPostition(containingSegment, localSeq) + containingOffset!;
     }
 
-    // Constructs an ITreeEntry for the cell data.
-    public snapshot(runtime: IFluidDataStoreRuntime, handle: IFluidHandle, serializer: IFluidSerializer): ITree {
-        return {
-            entries: [
-                {
-                    mode: FileMode.Directory,
-                    path: SnapshotPath.segments,
-                    type: TreeEntry.Tree,
-                    value: super.snapshot(runtime, handle, serializer, /* catchUpMsgs: */[]),
-                },
-                serializeBlob(handle, SnapshotPath.handleTable, this.handleTable.snapshot(), serializer),
-            ],
-        };
+    // Constructs an ISummaryTreeWithStats for the cell data.
+    public summarize(runtime: IFluidDataStoreRuntime, handle: IFluidHandle, serializer: IFluidSerializer):
+        ISummaryTreeWithStats {
+        const builder = new SummaryTreeBuilder();
+        builder.addWithStats(SnapshotPath.segments, super.summarize(runtime, handle, serializer, /* catchUpMsgs: */[]));
+        builder.addBlob(SnapshotPath.handleTable, serializer.stringify(this.handleTable.getSummaryContent(), handle));
+        return builder.getSummaryTree();
     }
 
     public async load(

packages/dds/merge-tree/package.json

@@ -60,6 +60,8 @@
     "@fluidframework/core-interfaces": "^0.41.0",
     "@fluidframework/datastore-definitions": "^0.55.0",
     "@fluidframework/protocol-definitions": "^0.1026.0",
+    "@fluidframework/runtime-definitions": "^0.55.0",
+    "@fluidframework/runtime-utils": "^0.55.0",
     "@fluidframework/shared-object-base": "^0.55.0",
     "@fluidframework/telemetry-utils": "^0.55.0"
   },

packages/dds/merge-tree/src/client.ts

@@ -7,8 +7,9 @@
 
 import { IFluidHandle } from "@fluidframework/core-interfaces";
 import { IFluidSerializer } from "@fluidframework/shared-object-base";
-import { ISequencedDocumentMessage, ITree, MessageType } from "@fluidframework/protocol-definitions";
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
 import { IFluidDataStoreRuntime, IChannelStorageService } from "@fluidframework/datastore-definitions";
+import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, Trace } from "@fluidframework/common-utils";
 import { LoggingError } from "@fluidframework/telemetry-utils";
@@ -874,29 +875,27 @@ export class Client {
         return new MergeTreeTextHelper(this.mergeTree);
     }
 
-    // TODO: Remove `catchUpMsgs` once new snapshot format is adopted as default.
-    //       (See https://github.com/microsoft/FluidFramework/issues/84)
-    public snapshot(
+    public summarize(
         runtime: IFluidDataStoreRuntime,
         handle: IFluidHandle,
         serializer: IFluidSerializer,
         catchUpMsgs: ISequencedDocumentMessage[],
-    ): ITree {
+    ): ISummaryTreeWithStats {
         const deltaManager = runtime.deltaManager;
         const minSeq = deltaManager.minimumSequenceNumber;
 
         // Catch up to latest MSN, if we have not had a chance to do it.
         // Required for case where FluidDataStoreRuntime.attachChannel()
-        // generates snapshot right after loading data store.
+        // generates summary right after loading data store.
 
         this.updateSeqNumbers(minSeq, deltaManager.lastSequenceNumber);
 
-        // One of the snapshots (from SPO) I observed to have chunk.chunkSequenceNumber > minSeq!
+        // One of the summaries (from SPO) I observed to have chunk.chunkSequenceNumber > minSeq!
         // Not sure why - need to catch it sooner
         assert(this.getCollabWindow().minSeq === minSeq,
             0x03e /* "minSeq mismatch between collab window and delta manager!" */);
 
-        // TODO: Remove options flag once new snapshot format is adopted as default.
+        // Must continue to support legacy
         //       (See https://github.com/microsoft/FluidFramework/issues/84)
         if (this.mergeTree.options?.newMergeTreeSnapshotFormat === true) {
             assert(