Automation: Main Next Integrate

packages/dds/tree/README.md

@@ -67,7 +67,20 @@ This typically means the client side "business logic" or "view" part of some gra
 
 ### Ownership and Lifetimes
 
-TODO: add a diagram for this section.
+This diagram shows the ownership hierarchy during a transaction with solid arrows, and some important references with dashed arrows:
+
+```mermaid
+graph TD;
+    store["Data Store"]-->doc["Persisted Summaries"]
+    container["Fluid Container"]-->shared-tree;
+    shared-tree--"extends"-->shared-tree-core;
+    shared-tree-core-."reads".->doc;
+    shared-tree-core-->EditManager-->X["collab window & branches"];
+    shared-tree-core-->Indexes-->ForestIndex;
+    shared-tree-->checkout["default checkout"]-->transaction-."updates".->checkout;
+    transaction-->ProgressiveEditBuilder
+    checkout-."reads".->ForestIndex;
+```
 
 `tree` is a DDS, and therefore it stores its persisted data in a Fluid Container, and is also owned by that same container.
 When nothing in that container references the DDS anymore, it may get garbage collected by the Fluid GC.
@@ -97,7 +110,14 @@ could be added in the future.
 
 #### Viewing
 
-TODO: add a diagram for this section.
+```mermaid
+graph LR;
+    doc["Persisted Summaries"]--"Summary+Trailing ops"-->shared-tree-core
+    shared-tree--"configures"-->shared-tree-core
+    shared-tree-core--"Summary"-->Indexes--"Summary"-->ForestIndex;
+    ForestIndex--"Exposed by"-->checkout;
+    checkout--"viewed by"-->app
+```
 
 [`shared-tree`](./src/shared-tree/) configures [`shared-tree-core`](./src/shared-tree-core/README.md) with a set of indexes.
 `shared-tree-core` downloads the summary data from the Fluid Container, feeding the summary data (and any future edits) into the indexes.

packages/dds/tree/src/changeset/format.ts

@@ -15,16 +15,14 @@ import { JsonableTree } from "../tree";
 export namespace Transposed {
 	export interface Transaction extends PeerChangeset {
 		/**
-		 * The reference sequence number of the transaction that this transaction was originally
-		 * issued after.
+		 * The tag of the changeset that this transaction was originally issued after.
 		 */
-		ref: SeqNumber;
+		ref: ChangesetTag;
 		/**
-		 * The reference sequence number of the transaction that this transaction has been
-		 * transposed over.
+		 * The tag of the latest changeset that this transaction has been transposed over.
 		 * Omitted on changesets that have not been transposed.
 		 */
-		newRef?: SeqNumber;
+		newRef?: ChangesetTag;
 	}
 
 	/**
@@ -53,41 +51,43 @@ export namespace Transposed {
 	export type MarkList<TMark = Mark> = TMark[];
 
 	export type Mark =
+		| SizedMark
+		| AttachGroup;
+
+	export type ObjectMark =
+		| SizedObjectMark
+		| AttachGroup;
+
+	export type SizedMark =
 		| Skip
+		| SizedObjectMark;
+
+	export type SizedObjectMark =
 		| Tomb
 		| Modify
 		| Detach
 		| Reattach
 		| ModifyReattach
 		| ModifyDetach
-		| GapEffectSegment
-		| AttachGroup;
+		| GapEffectSegment;
 
 	export type AttachGroup = Attach[];
 
 	export interface Tomb {
 		type: "Tomb";
-		seq: SeqNumber;
+		change: ChangesetTag;
 		count: number;
 	}
 
-	export type ValueMark = SetValue | RevertValue;
-
-	export interface SetValue {
-		type: "Set";
+	export interface SetValue extends HasOpId {
 		/** Can be left unset to represent the value being cleared. */
 		value?: Value;
 	}
 
-	export interface RevertValue {
-		type: "Revert";
-		seq: SeqNumber;
-	}
-
 	export interface Modify {
 		type: "Modify";
-		tomb?: SeqNumber;
-		value?: ValueMark;
+		tomb?: ChangesetTag;
+		value?: SetValue;
 		fields?: FieldMarks;
 	}
 
@@ -155,7 +155,7 @@ export namespace Transposed {
 	export interface ModifyInsert extends HasOpId, HasPlaceFields {
 		type: "MInsert";
 		content: ProtoNode;
-		value?: ValueMark;
+		value?: SetValue;
 		fields?: FieldMarks;
 	}
 
@@ -177,9 +177,9 @@ export namespace Transposed {
 	}
 
 	/**
-	 * Represents the precise location of a concurrent slice-move-in.
+	 * Represents the precise location of a concurrent slice-move-in within the same gap.
 	 * This is needed so we can tell where concurrent sliced-inserts (that this changeset has yet to be rebased over)
-	 * may land in the field. Without this, we would need to be able to retain information about the relative order in
+	 * may land in the gap. Without this, we would need to be able to retain information about the relative order in
 	 * time of any number of concurrent slice-moves. See scenario N.
 	 */
 	export interface Intake extends PriorOp {
@@ -196,7 +196,7 @@ export namespace Transposed {
 
 	export interface ModifyMoveIn extends HasOpId, HasPlaceFields {
 		type: "MMoveIn";
-		value?: ValueMark;
+		value?: SetValue;
 		fields?: FieldMarks;
 	}
 
@@ -207,7 +207,7 @@ export namespace Transposed {
 	export type GapEffectType = GapEffect["type"];
 
 	export interface GapEffectSegment {
-		tombs?: SeqNumber;
+		tomb?: ChangesetTag;
 		type: "Gap";
 		count: GapCount;
 		/**
@@ -235,28 +235,28 @@ export namespace Transposed {
 	export type NodeMark = Detach | Reattach;
 
 	export interface Detach extends HasOpId {
-		tomb?: SeqNumber;
+		tomb?: ChangesetTag;
 		gaps?: GapEffect[];
 		type: "Delete" | "MoveOut";
 		count: NodeCount;
 	}
 
 	export interface ModifyDetach extends HasOpId {
 		type: "MDelete" | "MMoveOut";
-		tomb?: SeqNumber;
-		value?: ValueMark;
+		tomb?: ChangesetTag;
+		value?: SetValue;
 		fields?: FieldMarks;
 	}
 
 	export interface Reattach extends HasOpId {
 		type: "Revive" | "Return";
-		tomb: SeqNumber;
+		tomb: ChangesetTag;
 		count: NodeCount;
 	}
 	export interface ModifyReattach extends HasOpId {
 		type: "MRevive" | "MReturn";
-		tomb: SeqNumber;
-		value?: ValueMark;
+		tomb: ChangesetTag;
+		value?: SetValue;
 		fields?: FieldMarks;
 	}
 
@@ -271,30 +271,13 @@ export namespace Transposed {
 	 */
 	export interface Tombstones {
 		count: NodeCount;
-		seq: PriorSeq;
+		change: ChangesetTag;
 	}
 
 	export interface PriorOp {
-		seq: PriorSeq;
+		change: ChangesetTag;
 		id: OpId;
 	}
-
-	/**
-	 * The sequence number of the edit that caused the nodes to be detached.
-	 *
-	 * When the nodes were detached as the result of learning of a prior concurrent change
-	 * that preceded a prior change that the current change depends on, a pair of sequence
-	 * numbers is used instead were `seq[0]` is the earlier change whose effect on `seq[1]`
-	 * these tombstones represent. This can be read as "tombstones from the effect of `seq[0]`
-	 * on `seq[1]`".
-	 */
-	export type PriorSeq = SeqNumber | [SeqNumber, SeqNumber];
-}
-
-export namespace Sequenced {
-	export interface Transaction extends Transposed.Transaction {
-		seq: SeqNumber;
-	}
 }
 
 export interface HasLength {
@@ -318,20 +301,12 @@ export enum RangeType {
 /**
  * A monotonically increasing positive integer assigned to each change within the changeset.
  * OpIds are scoped to a single changeset, so referring to OpIds across changesets requires
- * qualifying them by sequence/commit number.
+ * qualifying them by change tag.
  *
  * The uniqueness of IDs is leveraged to uniquely identify the matching move-out for a move-in/return and vice-versa.
  */
 export type OpId = number;
 
-export interface HasSeqNumber {
-	/**
-	 * Included in a mark to indicate the transaction it was part of.
-	 * This number is assigned by the Fluid service.
-	 */
-	seq: SeqNumber;
-}
-
 export interface HasOpId {
 	/**
 	 * The sequential ID assigned to a change within a transaction.
@@ -347,7 +322,7 @@ export type ProtoNode = JsonableTree;
 export type NodeCount = number;
 export type GapCount = number;
 export type Skip = number;
-export type SeqNumber = number;
+export type ChangesetTag = number | string;
 export type Value = number | string | boolean;
 export type ClientId = number;
 export enum Tiebreak { Left, Right }

packages/dds/tree/src/changeset/index.ts

@@ -9,3 +9,5 @@
 
 export * from "./format";
 export * from "./toDelta";
+export * from "./utils";
+export * from "./markListFactory";

packages/dds/tree/src/changeset/markListFactory.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { Skip, Transposed as T } from "./format";
+import { extendAttachGroup, isAttachGroup, isObjMark, isSkipMark, tryExtendMark } from "./utils";
+
+/**
+ * Helper class for constructing an offset list of marks that...
+ *  - Does not insert offsets if there is no content after them
+ *  - Does not insert 0-sized offsets
+ *  - Merges runs of offsets together
+ *  - Merges marks together
+ */
+export class MarkListFactory {
+    private offset = 0;
+    public readonly list: T.MarkList = [];
+
+    public push(...marks: T.Mark[]): void {
+        for (const item of marks) {
+            if (isSkipMark(item)) {
+                this.pushOffset(item);
+            } else {
+                this.pushContent(item);
+            }
+        }
+    }
+
+    public pushOffset(offset: Skip): void {
+        this.offset += offset;
+    }
+
+    public pushContent(mark: T.ObjectMark): void {
+        if (this.offset > 0) {
+            this.list.push(this.offset);
+            this.offset = 0;
+        }
+        const prev = this.list[this.list.length - 1];
+        if (isObjMark(prev)) {
+            if (isAttachGroup(prev)) {
+                if (isAttachGroup(mark)) {
+                    extendAttachGroup(prev, mark);
+                    return;
+                }
+            } else if (
+                !isAttachGroup(mark)
+                && prev.type === mark.type
+            ) {
+                // Neither are attach groups
+                if (tryExtendMark(prev, mark)) {
+                    return;
+                }
+            }
+        }
+        this.list.push(mark);
+    }
+}

packages/dds/tree/src/changeset/toDelta.ts

@@ -202,15 +202,7 @@ function applyOrCollectModifications(
 ): InsertedFieldsMarksMap {
     const outFieldsMarks: InsertedFieldsMarksMap = new Map();
     if (modify.value !== undefined) {
-        const type = modify.value.type;
-        switch (type) {
-            case "Set":
-                node.value = modify.value.value;
-                break;
-            case "Revert":
-                fail(ERR_REVERT_ON_INSERT);
-            default: unreachableCase(type);
-        }
+        node.value = modify.value.value;
     }
     if (modify.fields !== undefined) {
         const protoFields = node.fields ?? {};
@@ -328,7 +320,6 @@ function applyOrCollectModifications(
 const ERR_NOT_IMPLEMENTED = "Not implemented";
 const ERR_TOMB_IN_INSERT = "Encountered a concurrent deletion in inserted content";
 const ERR_MOD_ON_MISSING_FIELD = "Encountered a modification that targets a non-existent field on an inserted tree";
-const ERR_REVERT_ON_INSERT = "Encountered a revert operation on an inserted node";
 const ERR_BOUNCE_ON_INSERT = "Encountered a Bounce mark in an inserted field";
 const ERR_INTAKE_ON_INSERT = "Encountered an Intake mark in an inserted field";
 const ERR_REVIVE_ON_INSERT = "Encountered a Revive mark in an inserted field";
@@ -338,7 +329,7 @@ const ERR_RETURN_ON_INSERT = "Encountered a Return mark in an inserted field";
  * Modifications to a subtree as described by a Changeset.
  */
 interface ChangesetMods {
-    value?: T.ValueMark;
+    value?: T.SetValue;
     fields?: T.FieldMarks;
 }
 
@@ -356,15 +347,7 @@ interface ChangesetMods {
 function convertModify<TMarks>(modify: ChangesetMods): DeltaMods<TMarks> {
     const out: DeltaMods<TMarks> = {};
     if (modify.value !== undefined) {
-        const type = modify.value.type;
-        switch (type) {
-            case "Set":
-                out.setValue = modify.value.value;
-                break;
-            case "Revert":
-                fail(ERR_NOT_IMPLEMENTED);
-            default: unreachableCase(type);
-        }
+        out.setValue = modify.value.value;
     }
     const fields = modify.fields;
     if (fields !== undefined) {