Rewrite ViewSchema.checkCompatibility to leverage discrepancies (#23192)

## Description

Updates `ViewSchema.checkCompatibility` to leverage the
`getFieldDiscrepancies` codepath.

This also puts in place infrastructure to allow viewing the document
when the only differences between view and stored schema are extra
optional fields in the stored schema on nodes that the view schema
author has declared are OK.

---------

Co-authored-by: Abram Sanderson <absander@microsoft.com>

examples/apps/tree-cli-app/src/test/legacy/index.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+// Using 'export *' in this file increases readability of relevant tests by grouping schema according to their versions.
+// The usual concerns of export * around bundling do not apply.
+/* eslint-disable no-restricted-syntax */
+export * as v1 from "./v1.js";

examples/apps/tree-cli-app/src/test/legacy/v1.ts

@@ -0,0 +1,13 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { SchemaFactory } from "@fluidframework/tree";
+
+const schemaBuilder = new SchemaFactory("com.fluidframework.example.cli");
+
+/**
+ * List node.
+ */
+export class List extends schemaBuilder.array("List", [schemaBuilder.string]) {}

examples/apps/tree-cli-app/src/test/schema.spec.ts

@@ -11,63 +11,28 @@ import {
 	typeboxValidator,
 	type ForestOptions,
 	type ICodecOptions,
+	type ImplicitFieldSchema,
 	type JsonCompatible,
 	// eslint-disable-next-line import/no-internal-modules
 } from "@fluidframework/tree/alpha";
 
 import { List } from "../schema.js";
 
-// This file demonstrates how applications can write tests which ensure they maintain compatibility with the schema from previously released versions.
-
-describe("schema", () => {
-	it("current schema matches latest historical schema", () => {
-		const current = extractPersistedSchema(List);
-
-		// For compatibility with deep equality and simple objects, round trip via JSON to erase prototypes.
-		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-		const currentRoundTripped: JsonCompatible = JSON.parse(JSON.stringify(current));
-
-		const previous = historicalSchema.at(-1);
-		assert(previous !== undefined);
-		// This ensures that historicalSchema's last entry is up to date with the current application code.
-		// This can catch:
-		// 1. Forgetting to update historicalSchema when intentionally making schema changes.
-		// 2. Accidentally changing schema in a way that impacts document compatibility.
-		assert.deepEqual(currentRoundTripped, previous.schema);
-	});
-
-	it("historical schema can be upgraded to current schema", () => {
-		const options: ForestOptions & ICodecOptions = { jsonValidator: typeboxValidator };
-
-		for (let documentIndex = 0; documentIndex < historicalSchema.length; documentIndex++) {
-			for (let viewIndex = 0; viewIndex < historicalSchema.length; viewIndex++) {
-				const compat = comparePersistedSchema(
-					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-					historicalSchema[documentIndex]!.schema,
-					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-					historicalSchema[viewIndex]!.schema,
-					options,
-					false,
-				);
+import { v1 } from "./legacy/index.js";
 
-				// We do not expect duplicates in historicalSchema.
-				assert.equal(compat.isEquivalent, documentIndex === viewIndex);
-				// Currently collaboration is only allowed between identical versions
-				assert.equal(compat.canView, documentIndex === viewIndex);
-				// Older versions should be upgradable to newer versions, but not the reverse.
-				assert.equal(compat.canUpgrade, documentIndex <= viewIndex);
-			}
-		}
-	});
-});
+// This file demonstrates how applications can write tests which ensure they maintain compatibility with the schema from previously released versions.
 
 /**
  * List of schema from previous versions of this application.
  * Storing these as .json files in a folder may make more sense for more complex applications.
  *
  * The `schema` field is generated by passing the schema to `extractPersistedSchema`.
  */
-const historicalSchema: { version: string; schema: JsonCompatible }[] = [
+const historicalSchema: {
+	version: string;
+	schema: JsonCompatible;
+	viewSchema: ImplicitFieldSchema;
+}[] = [
 	{
 		version: "1.0",
 		schema: {
@@ -90,6 +55,7 @@ const historicalSchema: { version: string; schema: JsonCompatible }[] = [
 				types: ["com.fluidframework.example.cli.List"],
 			},
 		},
+		viewSchema: v1.List,
 	},
 	{
 		version: "2.0",
@@ -140,5 +106,51 @@ const historicalSchema: { version: string; schema: JsonCompatible }[] = [
 				types: ["com.fluidframework.example.cli.List"],
 			},
 		},
+		viewSchema: List,
 	},
 ];
+
+describe("schema", () => {
+	it("current schema matches latest historical schema", () => {
+		const current = extractPersistedSchema(List);
+
+		// For compatibility with deep equality and simple objects, round trip via JSON to erase prototypes.
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		const currentRoundTripped: JsonCompatible = JSON.parse(JSON.stringify(current));
+
+		const previous = historicalSchema.at(-1);
+		assert(previous !== undefined);
+		// This ensures that historicalSchema's last entry is up to date with the current application code.
+		// This can catch:
+		// 1. Forgetting to update historicalSchema when intentionally making schema changes.
+		// 2. Accidentally changing schema in a way that impacts document compatibility.
+		assert.deepEqual(currentRoundTripped, previous.schema);
+	});
+
+	describe("historical schema can be upgraded to current schema", () => {
+		const options: ForestOptions & ICodecOptions = { jsonValidator: typeboxValidator };
+
+		for (let documentIndex = 0; documentIndex < historicalSchema.length; documentIndex++) {
+			for (let viewIndex = 0; viewIndex < historicalSchema.length; viewIndex++) {
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				it(`document ${historicalSchema[documentIndex]!.version} vs view version ${historicalSchema[viewIndex]!.version}`, () => {
+					const compat = comparePersistedSchema(
+						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+						historicalSchema[documentIndex]!.schema,
+						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+						historicalSchema[viewIndex]!.viewSchema,
+						options,
+						false,
+					);
+
+					// We do not expect duplicates in historicalSchema.
+					assert.equal(compat.isEquivalent, documentIndex === viewIndex);
+					// Currently collaboration is only allowed between identical versions
+					assert.equal(compat.canView, documentIndex === viewIndex);
+					// Older versions should be upgradable to newer versions, but not the reverse.
+					assert.equal(compat.canUpgrade, documentIndex <= viewIndex);
+				});
+			}
+		}
+	});
+});

packages/dds/tree/api-report/tree.alpha.api.md

@@ -58,7 +58,7 @@ export interface CommitMetadata {
 }
 
 // @alpha
-export function comparePersistedSchema(persisted: JsonCompatible, view: JsonCompatible, options: ICodecOptions, canInitialize: boolean): SchemaCompatibilityStatus;
+export function comparePersistedSchema(persisted: JsonCompatible, view: ImplicitFieldSchema, options: ICodecOptions, canInitialize: boolean): SchemaCompatibilityStatus;
 
 // @alpha
 export type ConciseTree<THandle = IFluidHandle> = Exclude<TreeLeafValue, IFluidHandle> | THandle | ConciseTree<THandle>[] | {

packages/dds/tree/src/core/index.ts

@@ -207,7 +207,6 @@ export {
 export {
 	type Adapters,
 	AdaptedViewSchema,
-	Compatibility,
 	type TreeAdapter,
 	AllowedUpdateType,
 } from "./schema-view/index.js";

packages/dds/tree/src/core/schema-stored/schema.ts

@@ -92,6 +92,16 @@ export interface SchemaPolicy {
 	 * If true, new content inserted into the tree should be validated against the stored schema.
 	 */
 	readonly validateSchema: boolean;
+
+	/**
+	 * Whether to allow a document to be opened when a particular stored schema (identified by `identifier`)
+	 * contains optional fields that are not known to the view schema.
+	 *
+	 * @privateRemarks
+	 * Plumbing this in via `SchemaPolicy` avoids needing to walk the view schema representation repeatedly in places
+	 * that need it (schema validation, view vs stored compatibility checks).
+	 */
+	allowUnknownOptionalFields(identifier: TreeNodeSchemaIdentifier): boolean;
 }
 
 /**

packages/dds/tree/src/core/schema-view/index.ts

@@ -5,7 +5,6 @@
 
 export {
 	type Adapters,
-	Compatibility,
 	type TreeAdapter,
 	AdaptedViewSchema,
 	AllowedUpdateType,

packages/dds/tree/src/core/schema-view/view.ts

@@ -9,17 +9,6 @@ import type { TreeNodeSchemaIdentifier, TreeStoredSchema } from "../schema-store
  * APIs for applying `view schema` to documents.
  */
 
-/**
- * How compatible a particular view schema is for some operation on some specific document.
- */
-export enum Compatibility {
-	Incompatible,
-	// For write compatibility this can include compatible schema updates to stored schema.
-	// TODO: separate schema updates from adapters.
-	// RequiresAdapters,
-	Compatible,
-}
-
 /**
  * What kinds of updates to stored schema to permit.
  *

packages/dds/tree/src/feature-libraries/default-schema/defaultSchema.ts

@@ -13,4 +13,5 @@ import { fieldKinds } from "./defaultFieldKinds.js";
 export const defaultSchemaPolicy: FullSchemaPolicy = {
 	fieldKinds,
 	validateSchema: false,
+	allowUnknownOptionalFields: () => false,
 };

packages/dds/tree/src/feature-libraries/default-schema/schemaChecker.ts

@@ -73,7 +73,10 @@ export function isNodeInSchema(
 			uncheckedFieldsFromNode.delete(fieldKey);
 		}
 		// The node has fields that we did not check as part of looking at every field defined in the node's schema
-		if (uncheckedFieldsFromNode.size !== 0) {
+		if (
+			uncheckedFieldsFromNode.size !== 0 &&
+			!schemaAndPolicy.policy.allowUnknownOptionalFields(node.type)
+		) {
 			return SchemaValidationErrors.ObjectNode_FieldNotInSchema;
 		}
 	} else if (schema instanceof MapNodeStoredSchema) {

packages/dds/tree/src/feature-libraries/index.ts

@@ -76,7 +76,20 @@ export {
 	type FieldKindConfigurationEntry,
 	getAllowedContentDiscrepancies,
 	isRepoSuperset,
+	type AllowedTypeDiscrepancy,
+	type FieldKindDiscrepancy,
+	type ValueSchemaDiscrepancy,
+	type FieldDiscrepancy,
+	type NodeDiscrepancy,
+	type NodeKindDiscrepancy,
+	type NodeFieldsDiscrepancy,
 	isNeverTree,
+	type LinearExtension,
+	type Realizer,
+	fieldRealizer,
+	PosetComparisonResult,
+	comparePosetElements,
+	posetLte,
 } from "./modular-schema/index.js";
 
 export { mapRootChanges } from "./deltaUtils.js";