Add CI check to ensure SO mapping addition are done correctly

packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts

@@ -42,7 +42,14 @@ export type {
   MigrationStatus,
   MigrateDocumentOptions,
 } from './src/migration';
-export { parseObjectKey, getObjectKey, getIndexForType } from './src/utils';
+export {
+  parseObjectKey,
+  getObjectKey,
+  getIndexForType,
+  getFieldListFromTypeMapping,
+  getFieldListMapFromMappingDefinitions,
+  type FieldListMap,
+} from './src/utils';
 export {
   modelVersionVirtualMajor,
   globalSwitchToModelVersionAt,
@@ -68,4 +75,6 @@ export {
   buildModelVersionTransformFn,
   aggregateMappingAdditions,
   convertModelVersionBackwardConversionSchema,
+  getVersionAddedMappings,
+  getVersionAddedFields,
 } from './src/model_version';

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts

@@ -37,3 +37,4 @@ export { getModelVersionDelta } from './get_version_delta';
 export { buildModelVersionTransformFn } from './build_transform_fn';
 export { aggregateMappingAdditions } from './aggregate_model_changes';
 export { convertModelVersionBackwardConversionSchema } from './backward_conversion_schema';
+export { getVersionAddedFields, getVersionAddedMappings } from './version_mapping_changes';

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_mapping_changes.test.ts

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type {
+  SavedObjectsModelVersion,
+  SavedObjectsModelChange,
+} from '@kbn/core-saved-objects-server';
+import { getVersionAddedMappings, getVersionAddedFields } from './version_mapping_changes';
+
+const createVersion = (changes: SavedObjectsModelChange[]): SavedObjectsModelVersion => {
+  return {
+    changes,
+  };
+};
+
+describe('getVersionAddedMappings', () => {
+  it('returns empty mappings when the version has no changes', () => {
+    const version = createVersion([]);
+    expect(getVersionAddedMappings(version)).toEqual({});
+  });
+
+  it('returns empty mappings when the version has no `mappings_addition` changes', () => {
+    const version = createVersion([
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+    ]);
+    expect(getVersionAddedMappings(version)).toEqual({});
+  });
+
+  it(`returns the change's mappings when the version has a single 'mappings_addition' changes`, () => {
+    const version = createVersion([
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          nested: {
+            properties: {
+              foo: { type: 'text' },
+            },
+          },
+        },
+      },
+    ]);
+    expect(getVersionAddedMappings(version)).toEqual({
+      nested: {
+        properties: {
+          foo: { type: 'text' },
+        },
+      },
+    });
+  });
+
+  it(`merges the mappings when the version has multiple 'mappings_addition' changes`, () => {
+    const version = createVersion([
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          top: { type: 'text' },
+          nested: {
+            properties: {
+              bar: { type: 'text' },
+            },
+          },
+        },
+      },
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          nested: {
+            properties: {
+              foo: { type: 'text' },
+            },
+          },
+        },
+      },
+    ]);
+    expect(getVersionAddedMappings(version)).toEqual({
+      top: { type: 'text' },
+      nested: {
+        properties: {
+          foo: { type: 'text' },
+          bar: { type: 'text' },
+        },
+      },
+    });
+  });
+});
+
+describe('getVersionAddedFields', () => {
+  it('returns empty mappings when the version has no changes', () => {
+    const version = createVersion([]);
+    expect(getVersionAddedFields(version)).toEqual([]);
+  });
+
+  it('returns empty mappings when the version has no `mappings_addition` changes', () => {
+    const version = createVersion([
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+    ]);
+    expect(getVersionAddedFields(version)).toEqual([]);
+  });
+
+  it(`returns the change's mappings when the version has a single 'mappings_addition' changes`, () => {
+    const version = createVersion([
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          nested: {
+            properties: {
+              foo: { type: 'text' },
+            },
+          },
+        },
+      },
+    ]);
+    expect(getVersionAddedFields(version)).toEqual(['nested', 'nested.foo']);
+  });
+
+  it(`merges the mappings when the version has multiple 'mappings_addition' changes`, () => {
+    const version = createVersion([
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          top: { type: 'text' },
+          nested: {
+            properties: {
+              bar: { type: 'text' },
+            },
+          },
+        },
+      },
+      {
+        type: 'data_backfill',
+        backfillFn: jest.fn(),
+      },
+      {
+        type: 'mappings_addition',
+        addedMappings: {
+          nested: {
+            properties: {
+              foo: { type: 'text' },
+            },
+          },
+        },
+      },
+    ]);
+    expect(getVersionAddedFields(version)).toEqual(['nested', 'nested.bar', 'nested.foo', 'top']);
+  });
+});

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_mapping_changes.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { merge } from 'lodash';
+import type {
+  SavedObjectsMappingProperties,
+  SavedObjectsModelVersion,
+  SavedObjectsModelMappingsAdditionChange,
+} from '@kbn/core-saved-objects-server';
+import { getFieldListFromTypeMapping } from '../utils/get_field_list';
+
+/**
+ * Return the mappings that were introduced in the given version.
+ * If multiple 'mappings_addition' changes are present for the version,
+ * they will be deep-merged.
+ */
+export const getVersionAddedMappings = (
+  version: SavedObjectsModelVersion
+): SavedObjectsMappingProperties => {
+  const mappingChanges = version.changes.filter(
+    (change) => change.type === 'mappings_addition'
+  ) as SavedObjectsModelMappingsAdditionChange[];
+  return merge({}, ...mappingChanges.map((change) => change.addedMappings));
+};
+
+/**
+ * Return the list of fields, sorted, that were introduced in the given version.
+ */
+export const getVersionAddedFields = (version: SavedObjectsModelVersion): string[] => {
+  const addedMappings = getVersionAddedMappings(version);
+  return getFieldListFromTypeMapping({ properties: addedMappings });
+};

packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_field_list.test.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SavedObjectsTypeMappingDefinition } from '@kbn/core-saved-objects-server';
+import { getFieldListFromTypeMapping } from './get_field_list';
+
+describe('getFieldListFromTypeMapping', () => {
+  it('returns an empty list for empty mappings', () => {
+    const mappings: SavedObjectsTypeMappingDefinition = {
+      properties: {},
+    };
+    expect(getFieldListFromTypeMapping(mappings)).toEqual([]);
+  });
+
+  it('returns the correct list for top level fields', () => {
+    const mappings: SavedObjectsTypeMappingDefinition = {
+      properties: {
+        foo: { type: 'text' },
+        bar: { type: 'text' },
+      },
+    };
+    expect(getFieldListFromTypeMapping(mappings)).toEqual(['bar', 'foo']);
+  });
+
+  it('returns the correct list for deep fields', () => {
+    const mappings: SavedObjectsTypeMappingDefinition = {
+      properties: {
+        foo: {
+          properties: {
+            hello: { type: 'text' },
+            dolly: { type: 'text' },
+          },
+        },
+        bar: { type: 'text' },
+      },
+    };
+    expect(getFieldListFromTypeMapping(mappings)).toEqual(['bar', 'foo', 'foo.dolly', 'foo.hello']);
+  });
+
+  it('returns the correct list for any depth', () => {
+    const mappings: SavedObjectsTypeMappingDefinition = {
+      properties: {
+        foo: {
+          properties: {
+            hello: { type: 'text' },
+            dolly: {
+              properties: {
+                far: { type: 'text' },
+              },
+            },
+          },
+        },
+        bar: { type: 'text' },
+      },
+    };
+    expect(getFieldListFromTypeMapping(mappings)).toEqual([
+      'bar',
+      'foo',
+      'foo.dolly',
+      'foo.dolly.far',
+      'foo.hello',
+    ]);
+  });
+});

packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_field_list.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { MappingProperty as EsMappingProperty } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  SavedObjectsTypeMappingDefinition,
+  SavedObjectsFieldMapping,
+} from '@kbn/core-saved-objects-server';
+import type { SavedObjectsTypeMappingDefinitions } from '../mappings';
+
+export type FieldListMap = Record<string, string[]>;
+
+/**
+ * Return the list of fields present in each individual type mappings present in the definition.
+ */
+export const getFieldListMapFromMappingDefinitions = (
+  mappings: SavedObjectsTypeMappingDefinitions
+): FieldListMap => {
+  return Object.entries(mappings).reduce<FieldListMap>((memo, [typeName, typeMappings]) => {
+    memo[typeName] = getFieldListFromTypeMapping(typeMappings);
+    return memo;
+  }, {});
+};
+
+type AnyFieldMapping = SavedObjectsFieldMapping | EsMappingProperty;
+
+interface QueueItem {
+  fieldPath: string[];
+  fieldDef: AnyFieldMapping;
+}
+
+/**
+ * Return the list of fields present in the provided mappings.
+ * Note that fields only containing properties are still considered fields by this function.
+ *
+ * @example
+ * ```
+ * getFieldListFromTypeMapping({
+ *   properties: {
+ *     foo: {
+ *       properties: {
+ *         hello: { type: 'text' },
+ *         dolly: { type: 'text' },
+ *       },
+ *     },
+ *   },
+ * });
+ * // ['foo', 'foo.dolly', 'foo.hello']
+ * ```
+ */
+export const getFieldListFromTypeMapping = (
+  typeMappings: SavedObjectsTypeMappingDefinition
+): string[] => {
+  const fieldList: string[] = [];
+  const queue: QueueItem[] = [];
+
+  Object.entries(typeMappings.properties).forEach(([fieldName, fieldDef]) => {
+    queue.push({
+      fieldPath: [fieldName],
+      fieldDef,
+    });
+  });
+
+  while (queue.length > 0) {
+    const item = queue.pop()!;
+    fieldList.push(item.fieldPath.join('.'));
+    if ('properties' in item.fieldDef) {
+      Object.entries(item.fieldDef.properties ?? {}).forEach(([fieldName, fieldDef]) => {
+        queue.push({
+          fieldPath: [...item.fieldPath, fieldName],
+          fieldDef,
+        });
+      });
+    }
+  }
+
+  return fieldList.sort();
+};

packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/index.ts

@@ -8,3 +8,8 @@
 
 export { getObjectKey, parseObjectKey } from './object_key';
 export { getIndexForType } from './get_index_for_type';
+export {
+  getFieldListFromTypeMapping,
+  getFieldListMapFromMappingDefinitions,
+  type FieldListMap,
+} from './get_field_list';