From 5aa9983a3f40dff954ef162ff9e31dfe4f6fd85a Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 2 Feb 2023 10:17:29 +0100 Subject: [PATCH 01/13] Add the new types --- .../core-saved-objects-server/index.ts | 11 ++ .../src/model_version/index.ts | 22 ++++ .../src/model_version/migration.ts | 37 +++++++ .../src/model_version/model_version.ts | 95 +++++++++++++++++ .../src/saved_objects_type.ts | 100 ++++++++++++++++++ 5 files changed, 265 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts create mode 100644 packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts create mode 100644 packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index 33b9dfd64f6f8a..ebc933224b2c54 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -89,6 +89,17 @@ export { SECURITY_EXTENSION_ID, SPACES_EXTENSION_ID, } from './src/extensions/extensions'; +export type { + SavedObjectsModelVersion, + SavedObjectsModelVersionMap, + SavedObjectsModelVersionMapProvider, + SavedObjectsModelChange, + SavedObjectsModelExpansionChange, + SavedObjectModelMigrationDoc, + SavedObjectModelMigrationContext, + SavedObjectModelMigrationFn, + SavedObjectModelBidirectionalMigration, +} from './src/model_version'; export type { SavedObject, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts new file mode 100644 index 00000000000000..feb2feeb0e67e9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type { + SavedObjectModelMigrationDoc, + SavedObjectModelMigrationContext, + SavedObjectModelMigrationFn, + SavedObjectModelBidirectionalMigration, +} from './migration'; + +export type { + SavedObjectsModelChange, + SavedObjectsModelExpansionChange, + SavedObjectsModelVersion, + SavedObjectsModelVersionMap, + SavedObjectsModelVersionMapProvider, +} from './model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts new file mode 100644 index 00000000000000..9ecee81c18594b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectSanitizedDoc } from '../serialization'; +import type { SavedObjectsMigrationLogger } from '../migration'; + +// alias to more easily adapt later +export type SavedObjectModelMigrationDoc = SavedObjectSanitizedDoc; + +export type SavedObjectModelMigrationFn = ( + doc: SavedObjectModelMigrationDoc, + context: SavedObjectModelMigrationContext +) => SavedObjectModelMigrationDoc; + +export interface SavedObjectModelMigrationContext { + /** + * logger instance to be used by the migration handler + */ + readonly log: SavedObjectsMigrationLogger; + /** + * The model version this migration is registered for + */ + readonly modelVersion: number; +} + +export interface SavedObjectModelBidirectionalMigration< + PreviousAttributes = unknown, + NewAttributes = unknown +> { + up: SavedObjectModelMigrationFn; + down: SavedObjectModelMigrationFn; +} diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts new file mode 100644 index 00000000000000..a1f40963843e07 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsMappingProperties } from '../mapping_definition'; +import type { SavedObjectModelBidirectionalMigration } from './migration'; + +/** + * Represents a model version of a given savedObjects type. + * + * Model versions supersede the {@link SavedObjectsType.migrations | migrations} (and {@link SavedObjectsType.schemas | schemas}) APIs + * by exposing an unified way of describing the changes of shape or data of a type. + * + * @public + */ +export interface SavedObjectsModelVersion { + /** + * The {@link SavedObjectsModelChange | changes} associated with this version. + */ + modelChange: SavedObjectsModelChange; +} + +/** + * {@link SavedObjectsModelChange | model change} representing an expansion. + * + * A model expansion can do either, or both, or those: + * - add new mappings + * - migrate data in a backward-compatible way + * + * @remark when adding mappings, {@link SavedObjectsType.mappings | the type mappings} must also be updated accordingly. + * Overall, the type's mapping represents the latests version of the mappings, where the model changes + * represent the changes of mappings between two versions. + * + * @public + */ +export interface SavedObjectsModelExpansionChange< + PreviousAttributes = unknown, + NewAttributes = unknown +> { + /** + * The type of {@link SavedObjectsModelChange | change}, used to identify them internally. + */ + type: 'expansion'; + /** + * (optional) The new mappings introduced in this version. + */ + addedMappings?: SavedObjectsMappingProperties; + /** + * (optional) A bidirectional migration to migrate the data from and/or to the previous model version. + */ + migration?: SavedObjectModelBidirectionalMigration; +} + +/** + * Identify the model change associated with a given {@link SavedObjectsModelVersion}. + * + * At the moment, Only one type of change is supported: {@link SavedObjectsModelExpansionChange | expansions}. + * + * @public + */ +export type SavedObjectsModelChange = SavedObjectsModelExpansionChange; + +/** + * A record of {@link SavedObjectsModelVersion | model versions} for a given savedObjects type. + * The record's keys must be integers, starting with 1 for the first entry, and there shouldn't be gaps. + * + * @example + * ```typescript + * const modelVersionMap: SavedObjectsModelVersionMap = { + * '1': modelVersion1, + * '2': modelVersion2, + * '3': modelVersion3, + * } + * ``` + * + * @public + */ +export interface SavedObjectsModelVersionMap { + [modelVersion: string]: SavedObjectsModelVersion; +} + +/** + * A function returning a {@link SavedObjectsModelVersionMap | model version map} + * + * Ensured to be called after all plugins executed their `setup` phase. + * Similar to what was done with migrations, can be used to defer resolving the model versions + * associated to a type to after all plugins have been set up. + * + * @public + */ +export type SavedObjectsModelVersionMapProvider = () => SavedObjectsModelVersionMap; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts index 9dee63c3556769..370dbd8acb2186 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts @@ -14,6 +14,10 @@ import type { SavedObjectsTypeManagementDefinition } from './saved_objects_manag import type { SavedObjectsValidationMap } from './validation'; import type { SavedObjectMigrationMap } from './migration'; import type { SavedObjectsTypeMappingDefinition } from './mapping_definition'; +import type { + SavedObjectsModelVersionMap, + SavedObjectsModelVersionMapProvider, +} from './model_version'; /** * @public @@ -128,6 +132,102 @@ export interface SavedObjectsType { * An optional {@link SavedObjectsTypeManagementDefinition | saved objects management section} definition for the type. */ management?: SavedObjectsTypeManagementDefinition; + + /** + * A map of model versions associated with this type. + * + * Model versions supersede the {@link SavedObjectsType.migrations | migrations} (and {@link SavedObjectsType.schemas | schemas}) APIs + * by exposing an unified way of describing the changes of shape or data of the type. + * + * Model versioning is decoupled from Kibana versioning, and isolated between types. + * Model versions are identified by a single numeric value, starting at `1` and without gaps. + * + * Please refer to {@link SavedObjectsModelVersion} for details on the API's usages. + * + * A **valid** versioning would be: + * + * ```ts + * { + * name: 'foo', + * // other mandatory attributes... + * modelVersions: { + * '1': modelVersion1, + * '2': modelVersion2, + * '3': modelVersion3, + * } + * } + * ``` + * + * A **invalid** versioning would be: + * + * ```ts + * { + * name: 'foo', + * // other mandatory attributes... + * modelVersions: { + * '1': modelVersion1, + * '3': modelVersion3, // ERROR, no model version 2 + * '3.1': modelVersion31, // ERROR, model version is a single numeric value + * } + * } + * ``` + * + * @alpha experimental and subject to change. + */ + modelVersions?: SavedObjectsModelVersionMap | SavedObjectsModelVersionMapProvider; + + /** + * Allows to opt-in to the new model version API. + * + * Must be a valid semver version (with the patch version being necessarily 0) + * + * When specified, the type will switch from using the {@link SavedObjectsType.migrations | legacy migration API} + * to use the {@link SavedObjectsType.modelVersions | modelVersion API} after the specified version. + * + * When opted in, it will no longer be possible to use the legacy migration API after the specified version. + * + * A **valid** usage example would be: + * + * ```ts + * { + * name: 'foo', + * // other mandatory attributes... + * switchToModelVersionAt: '8.8.0', + * migrations: { + * '8.1.0': migrateTo810, + * '8.7.0': migrateTo870, + * }, + * modelVersions: { + * '1': modelVersion1 + * } + * } + * ``` + * + * An **invalid** usage example would be: + * + * ```ts + * { + * name: 'foo', + * // other mandatory attributes... + * switchToModelVersionAt: '8.9.0', + * migrations: { + * '8.1.0': migrateTo8_1, + * '8.9.0': migrateTo8_9, // error: migration registered for the switch version + * '8.10.0': migrateTo8_10, // error: migration registered for after the switch version + * }, + * modelVersions: { + * '1': modelVersion1 + * } + * } + * ``` + * + * Please refer to the {@link SavedObjectsType.modelVersions | modelVersion API} for more documentation on + * the new API. + * + * @remarks All types will be forced to switch to use the new API in a later version. This switch is + * allowing types owners to switch their types before the milestone. + */ + switchToModelVersionAfter?: string; } /** From fdb781185b446efb38a9a52fa0f5e2200ffda37a Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 2 Feb 2023 11:27:18 +0100 Subject: [PATCH 02/13] update migration hash helpers --- .../src/extract_migration_info.test.ts | 104 +++++++++++ .../src/extract_migration_info.ts | 26 +++ .../src/get_migration_hash.test.ts | 167 ++++++++++++++++++ .../src/get_migration_hash.ts | 13 +- 4 files changed, 309 insertions(+), 1 deletion(-) diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts index 47620aee90b3c3..65853ffc9494b0 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts @@ -158,4 +158,108 @@ describe('extractMigrationInfo', () => { }); }); }); + + describe('modelVersions', () => { + it('returns the correct switchToModelVersionAfter', () => { + const type = createType({ + switchToModelVersionAfter: '8.8.0', + }); + const output = extractMigrationInfo(type); + + expect(output.switchToModelVersionAfter).toEqual('8.8.0'); + }); + + it('returns a proper summary of the model versions', () => { + const type = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { + up: jest.fn(), + down: jest.fn(), + }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { + type: 'boolean', + }, + }, + }, + }, + }, + }); + const output = extractMigrationInfo(type); + + expect(output.modelVersions).toEqual([ + { + version: '1', + changeType: 'expansion', + hasMigration: true, + newMappings: [], + }, + { + version: '2', + changeType: 'expansion', + newMappings: ['foo.type'], + hasMigration: false, + }, + ]); + }); + + it('supports provider functions', () => { + const type = createType({ + modelVersions: () => ({ + '1': { + modelChange: { + type: 'expansion', + migration: { + up: jest.fn(), + down: jest.fn(), + }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { + type: 'boolean', + }, + }, + }, + }, + }), + }); + const output = extractMigrationInfo(type); + + expect(output.modelVersions).toEqual([ + { + version: '1', + changeType: 'expansion', + hasMigration: true, + newMappings: [], + }, + { + version: '2', + changeType: 'expansion', + newMappings: ['foo.type'], + hasMigration: false, + }, + ]); + }); + + it('returns an empty list when model versions are not defined', () => { + const type = createType({ + modelVersions: undefined, + }); + const output = extractMigrationInfo(type); + + expect(output.modelVersions).toEqual([]); + }); + }); }); diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts index c2f1c0b7aa9a2b..58779f0aace082 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts @@ -20,6 +20,15 @@ export interface SavedObjectTypeMigrationInfo { schemaVersions: string[]; mappings: Record; hasExcludeOnUpgrade: boolean; + modelVersions: ModelVersionSummary[]; + switchToModelVersionAfter?: string; +} + +export interface ModelVersionSummary { + version: string; + changeType: string; + hasMigration: boolean; + newMappings: string[]; } /** @@ -37,6 +46,21 @@ export const extractMigrationInfo = (soType: SavedObjectsType): SavedObjectTypeM const schemaVersions = Object.keys(schemaMap ?? {}); schemaVersions.sort(semverCompare); + const modelVersionMap = + typeof soType.modelVersions === 'function' + ? soType.modelVersions() + : soType.modelVersions ?? {}; + const modelVersionIds = Object.keys(modelVersionMap); + const modelVersions = modelVersionIds.map((version) => { + const entry = modelVersionMap[version]; + return { + version, + changeType: entry.modelChange.type, + hasMigration: !!entry.modelChange.migration, + newMappings: Object.keys(getFlattenedObject(entry.modelChange.addedMappings ?? {})), + }; + }); + return { name: soType.name, namespaceType: soType.namespaceType, @@ -46,5 +70,7 @@ export const extractMigrationInfo = (soType: SavedObjectsType): SavedObjectTypeM schemaVersions, mappings: getFlattenedObject(soType.mappings ?? {}), hasExcludeOnUpgrade: !!soType.excludeOnUpgrade, + modelVersions, + switchToModelVersionAfter: soType.switchToModelVersionAfter, }; }; diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts index 2ac9e04172e860..f1ff5510da8fb4 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts @@ -131,6 +131,7 @@ describe('getMigrationHash', () => { expect(getMigrationHash(typeA)).not.toEqual(getMigrationHash(typeB)); }); }); + describe('schemas', () => { it('returns same hash if same schema versions are registered', () => { const typeA = createType({ @@ -321,6 +322,172 @@ describe('getMigrationHash', () => { }); }); + describe('model versions', () => { + it('returns same hash if same model versions versions are registered', () => { + const typeA = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }, + }); + const typeB = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }, + }); + + expect(getMigrationHash(typeA)).toEqual(getMigrationHash(typeB)); + }); + + it('returns same hash if same model versions are registered in different order', () => { + const typeA = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }, + }); + const typeB = createType({ + modelVersions: { + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + }, + }); + + expect(getMigrationHash(typeA)).toEqual(getMigrationHash(typeB)); + }); + + it('returns same hash if same model versions are registered using record + function', () => { + const typeA = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }, + }); + const typeB = createType({ + modelVersions: () => ({ + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }), + }); + + expect(getMigrationHash(typeA)).toEqual(getMigrationHash(typeB)); + }); + + it('returns different hashes if different model versions are registered', () => { + const typeA = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { type: 'boolean' }, + }, + }, + }, + }, + }); + const typeB = createType({ + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + bar: { type: 'boolean' }, + }, + }, + }, + }, + }); + + expect(getMigrationHash(typeA)).not.toEqual(getMigrationHash(typeB)); + }); + }); + describe('ignored fields', () => { it('returns same hash if `hidden` changes', () => { expect(getMigrationHash(createType({ hidden: false }))).toEqual( diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts index 7e23ec35bb9e7d..b0123a39bc873c 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts @@ -8,7 +8,7 @@ import { createHash } from 'crypto'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { extractMigrationInfo } from './extract_migration_info'; +import { extractMigrationInfo, ModelVersionSummary } from './extract_migration_info'; type SavedObjectTypeMigrationHash = string; @@ -26,8 +26,19 @@ export const getMigrationHash = (soType: SavedObjectsType): SavedObjectTypeMigra migInfo.migrationVersions.join(','), migInfo.schemaVersions.join(','), JSON.stringify(migInfo.mappings, Object.keys(migInfo.mappings).sort()), + migInfo.switchToModelVersionAfter ?? 'none', + migInfo.modelVersions.map(serializeModelVersion).join(','), ]; const hashFeed = hashParts.join('-'); return hash.update(hashFeed).digest('hex'); }; + +const serializeModelVersion = (modelVersion: ModelVersionSummary): string => { + return [ + modelVersion.version, + modelVersion.changeType, + modelVersion.hasMigration, + ...modelVersion.newMappings, + ].join('|'); +}; From 56e07ed3c819bc6770976e23d62fe9b733ad3ed4 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 2 Feb 2023 11:34:13 +0100 Subject: [PATCH 03/13] updates the registered types test snapshots --- .../migrations/check_registered_types.test.ts | 194 +++++++++--------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index 347401e9b30f99..26b27ee25fc715 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -55,103 +55,103 @@ describe('checking migration metadata changes on all registered SO types', () => expect(hashMap).toMatchInlineSnapshot(` Object { - "action": "7858e6d5a9f231bf23f6f2e57328eb0095b26735", - "action_task_params": "bbd38cbfd74bf6713586fe078e3fa92db2234299", - "alert": "f2e81863be0b50966b876b88b906c962e30b8c9c", - "api_key_pending_invalidation": "9b4bc1235337da9a87ef05a1d1f4858b2a3b77c6", - "apm-indices": "ceb0870f3a74e2ffc3a1cd3a3c73af76baca0999", - "apm-server-schema": "2bfd2998d3873872e1366458ce553def85418f91", - "apm-service-group": "07ecbf25ee4828d2b686abc98656b6665831d1a0", - "apm-telemetry": "abaa1e9469e6e0bad76938309f0ac4c66b528d58", - "app_search_telemetry": "7fc4fc08852bf0924ee29942bb394fda9aa8954d", - "application_usage_daily": "6e645e0b60ef3af2e8fde80963c2a4f09a190d61", - "application_usage_totals": "b2af3577dcd50bfae492b166a7804f69e2cc41dc", - "canvas-element": "e2e312fc499c1a81e628b88baba492fb24f4e82d", - "canvas-workpad": "4b05f7829bc805bbaa07eb9fc0d2a2bbbd6bbf39", - "canvas-workpad-template": "d4bb65aa9c4a2b25029d3272fd9c715d8e4247d7", - "cases": "a27d57e75e358349a6ba835152fd4de0033a7bff", - "cases-comments": "d7c4c1d24e97620cd415e27e5eb7d5b5f2c5b461", - "cases-configure": "1afc414f5563a36e4612fa269193d3ed7277c7bd", - "cases-connector-mappings": "4b16d440af966e5d6e0fa33368bfa15d987a4b69", - "cases-telemetry": "16e261e7378a72acd0806f18df92525dd1da4f37", - "cases-user-actions": "f1b0dcfeb58a65e68b35c5e99ddee70e746a06c7", - "config": "e3f0408976dbdd453641f5699927b28b188f6b8c", - "config-global": "b8f559884931609a349e129c717af73d23e7bc76", - "connector_token": "fa5301aa5a2914795d3b1b82d0a49939444009da", - "core-usage-stats": "f40a213da2c597b0de94e364a4326a5a1baa4ca9", - "csp-rule-template": "d6104585d0b032355c64a7dbf2a834163351cb1c", - "dashboard": "7e37790f802b39c852f905c010e13674e893105a", - "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", - "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", - "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "21e096cf4554abe1652953a6cd2119d68ddc9403", - "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", - "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", - "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", - "exception-list-agnostic": "49fae8fcd1967cc4be45ba2a2c66c4afbc1e341b", - "file": "f5d393602a4c103eb0ace983e6810f7f3666544b", - "file-upload-usage-collection-telemetry": "8478924cf0057bd90df737155b364f98d05420a5", - "fileShare": "3f88784b041bb8728a7f40763a08981828799a75", - "fleet-fleet-server-host": "643d15dbf56edb96f7ca65f98409d83ac5792fb6", - "fleet-preconfiguration-deletion-record": "7b28f200513c28ae774f1b7d7d7906954e3c6e16", - "fleet-proxy": "2bbcd9e6d5e30ac07b275c8d489af07a0d550df5", - "graph-workspace": "3342f2cd561afdde8f42f5fb284bf550dee8ebb5", - "guided-onboarding-guide-state": "561db8d481b131a2bbf46b1e534d6ce960255135", - "guided-onboarding-plugin-state": "a802ed58e9d0076b9632c59d7943861ba476f99c", - "index-pattern": "48e77ca393c254e93256f11a7cdc0232dd754c08", - "infrastructure-monitoring-log-view": "e2c78c1076bd35e57d7c5fa1b410e5c126d12327", - "infrastructure-ui-source": "7c8dbbc0a608911f1b683a944f4a65383f6153ed", - "ingest-agent-policies": "54d586fdafae83ba326e47d1a3727b0d9c910a12", - "ingest-download-sources": "1e69dabd6db5e320fe08c5bda8f35f29bafc6b54", - "ingest-outputs": "29181ecfdc7723f544325ecef7266bccbc691a54", - "ingest-package-policies": "d93048bf153f9043946e8965065a88014f7ccb41", - "ingest_manager_settings": "6f36714825cc15ea8d7cda06fde7851611a532b4", - "inventory-view": "bc2bd1e7ec7c186159447ab228d269f22bd39056", - "kql-telemetry": "29544cd7d3b767c5399878efae6bd724d24c03fd", - "legacy-url-alias": "7172dfd54f2e0c89fe263fd7095519b2d826a930", - "lens": "236ecd358ed3a4ecfc03ed676d958b64acf0b697", - "lens-ui-telemetry": "df2844565c9e18fed2bdb1f6cc3aadd58cf1e45b", - "map": "00ca6c4cf46ae59f70f1436262eb9f457b45eb14", - "maps-telemetry": "5adbde35bd50ec2b8e9ea5b96d4d9f886e31ecfb", - "metrics-explorer-view": "09e56993352b8ee678e88f71e4410d9aeee72f3a", - "ml-job": "2836da98a81bd220db61c0549e8e28da7a876cb2", - "ml-module": "95055522c8406afa67a554690a43506f6c040744", - "ml-trained-model": "e39dd10b2da827e194ddcaaf3db141ad1daf0201", - "monitoring-telemetry": "af508cea8e22edaa909e462069390650fbbf01b7", - "osquery-manager-usage-metric": "fbe3cbea25a96e2ca522ca436878e0162c94dcc2", - "osquery-pack": "a2d675c7af4208e54a5b28d23d324d7c599a5491", - "osquery-pack-asset": "de8783298eb33a577bf1fa0caacd42121dcfae91", - "osquery-saved-query": "7b213b4b7a3e59350e99c50e8df9948662ed493a", - "query": "4640ef356321500a678869f24117b7091a911cb6", - "rules-settings": "1af4c9abd4b40a154e233c2af4867df7aab7ac24", - "sample-data-telemetry": "8b10336d9efae6f3d5593c4cc89fb4abcdf84e04", - "search": "c48f5ab5d94545780ea98de1bff9e39f17f3606b", - "search-session": "ba383309da68a15be3765977f7a44c84f0ec7964", - "search-telemetry": "beb3fc25488c753f2a6dcff1845d667558712b66", - "security-rule": "e0dfdba5d66139d0300723b2e6672993cd4a11f3", - "security-solution-signals-migration": "e65933e32926e0ca385415bd44fc6da0b6d3d419", - "siem-detection-engine-rule-actions": "d4b5934c0c0e4ccdf509a41000eb0bee07be0c28", - "siem-ui-timeline": "95474f10662802e2f9ea068b45bf69212a2f5842", - "siem-ui-timeline-note": "08c71dc0b8b8018a67e80beb4659a078404c223d", - "siem-ui-timeline-pinned-event": "e2697b38751506c7fce6e8b7207a830483dc4283", - "space": "c4a0acce1bd4b9cce85154f2a350624a53111c59", - "spaces-usage-stats": "922d3235bbf519e3fb3b260e27248b1df8249b79", - "synthetics-monitor": "7c1e5a78fb3b88cc03b441d3bf3714d9967ab214", - "synthetics-param": "53dee203042c238888247084336f2dba777f4a65", - "synthetics-privates-locations": "dd00385f4a27ef062c3e57312eeb3799872fa4af", - "tag": "39413f4578cc2128c9a0fda97d0acd1c8862c47a", - "task": "ef53d0f070bd54957b8fe22fae3b1ff208913f76", - "telemetry": "9142dc5f18123fb6e6a9083db04e5becbfde94fd", - "ui-metric": "2fb66ccdee2d1fad52547964421629c5a485c38f", - "upgrade-assistant-ml-upgrade-operation": "408120d386c04ab25fe64a03937597aa0438c10d", - "upgrade-assistant-reindex-operation": "d9e18b3d9578ecabf09a297296dcf7e36b2481fd", - "upgrade-assistant-telemetry": "a0c80933a9f8b50a2590d19e1d1e5f97d28f7104", - "uptime-dynamic-settings": "9de35c5aeaef915c5bc3c5b1632c33fb0f6f1c55", - "uptime-synthetics-api-key": "df9d8418ddc210d832a069a0fb796f73e63d1082", - "url": "d66c1f26ed23a392be3617a8444d713571f58380", - "usage-counters": "33e2081a52215293041da1100e6602fb553ff446", - "visualization": "f45d06858a5634c9ed0367e11eb44f7f7dde0be2", - "workplace_search_telemetry": "45bd03e12b060c08381b0fd325d939f80d08c914", + "action": "6cfc277ed3211639e37546ac625f4a68f2494215", + "action_task_params": "db2afea7d78e00e725486b791554d0d4e81956ef", + "alert": "492f749af60bed1cc7022519ddf5fe4c482b62c4", + "api_key_pending_invalidation": "16e7bcf8e78764102d7f525542d5b616809a21ee", + "apm-indices": "d19dd7fb51f2d2cbc1f8769481721e0953f9a6d2", + "apm-server-schema": "1d42f17eff9ec6c16d3a9324d9539e2d123d0a9a", + "apm-service-group": "2801c50332e3bf5726c923966c1f19f886e7d389", + "apm-telemetry": "712138c3d5e36d865e67c7fc0171c8a779446e58", + "app_search_telemetry": "9269643c9a5894998b44883f7f7d07a453fd6a95", + "application_usage_daily": "9867f6e1355124f822beab051e0fbac4cc117eac", + "application_usage_totals": "9469a48ab887761a73ee3719b8d401ac627f1eb1", + "canvas-element": "ec334dd45d14291db4d74197e0e42dfe06526868", + "canvas-workpad": "ab0525bd5aa4dbad2d6fdb30e6a51bb475254751", + "canvas-workpad-template": "c54f2a188a1d0bf18a6cebd9d6f28a7337d41bbf", + "cases": "74c00dfb25f4b109894971bd1090fce4a7c99490", + "cases-comments": "b94072c8f0cb642dd6da43c833bd2d391e175ec0", + "cases-configure": "25099c9e4bbb91e01e334848c605b4a5de5c9fce", + "cases-connector-mappings": "8de3b77dc6ae8ee62cce2b58a222471dfc3dbdad", + "cases-telemetry": "fdeddcef28c75d8c66422475a829e75d37f0b668", + "cases-user-actions": "ca0dbab8f8f169ef66f92570b45ec02e57a82d78", + "config": "0090bed77e173b5d2e4b13a60388c153910cd967", + "config-global": "d9791e8f73edee884630e1cb6e4954ae513ce75e", + "connector_token": "fb05ff5afdcb6e2f20c9c6513ff7a1ab12b66f36", + "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", + "csp-rule-template": "099c229bf97578d9ca72b3a672d397559b84ee0b", + "dashboard": "71e3f8dfcffeb5fbd410dec81ce46f5691763c43", + "endpoint:user-artifact": "a5b154962fb6cdf5d9e7452e58690054c95cc72a", + "endpoint:user-artifact-manifest": "5989989c0f84dd2d02da1eb46b6254e334bd2ccd", + "enterprise_search_telemetry": "4b41830e3b28a16eb92dee0736b44ae6276ced9b", + "epm-packages": "1922a722ea42ab4953a96037fabb81a9ded8e240", + "epm-packages-assets": "00c8b5e5bf059627ffc9fbde920e1ac75926c5f6", + "event_loop_delays_daily": "ef49e7f15649b551b458c7ea170f3ed17f89abd0", + "exception-list": "aae42e8f19017277d194d37d4898ed6598c03e9a", + "exception-list-agnostic": "2634ee4219d27663a5755536fc06cbf3bb4beba5", + "file": "d12998f49bc82da596a9e6c8397999930187ec6a", + "file-upload-usage-collection-telemetry": "c6fcb9a7efcf19b2bb66ca6e005bfee8961f6073", + "fileShare": "f07d346acbb724eacf139a0fb781c38dc5280115", + "fleet-fleet-server-host": "67180a54a689111fb46403c3603c9b3a329c698d", + "fleet-preconfiguration-deletion-record": "3afad160748b430427086985a3445fd8697566d5", + "fleet-proxy": "94d0a902a0fd22578d7d3a20873b95d902e25245", + "graph-workspace": "565642a208fe7413b487aea979b5b153e4e74abe", + "guided-onboarding-guide-state": "3257825ae840309cb676d64b347107db7b76f30a", + "guided-onboarding-plugin-state": "2d3ef3069ca8e981cafe8647c0c4a4c20739db10", + "index-pattern": "cd51191712081278c2af83d16552c3438ef83353", + "infrastructure-monitoring-log-view": "8040108f02ef27419cff79077384379709d44bbc", + "infrastructure-ui-source": "2311f7d0abe2a713aa71e30ee24f78828d4acfc1", + "ingest-agent-policies": "deff33c5b5150296c79e52dd1f4905ef189e7517", + "ingest-download-sources": "95a15b6589ef46e75aca8f7e534c493f99cc3ccd", + "ingest-outputs": "f5adeb3f6abc732a6067137e170578dbf1f58c62", + "ingest-package-policies": "af65ee5c419bcf8ecc05ec2d74b3662f0fb30d21", + "ingest_manager_settings": "fb75bff08a8de3435b23664b1191f9244a255701", + "inventory-view": "6d47ef0b38166ecbd1c2fc7394599a4500db1ae4", + "kql-telemetry": "23ed96ff02cd69cbfaa22f313cae3a54c434db51", + "legacy-url-alias": "9b8cca3fbb2da46fd12823d3cd38fdf1c9f24bc8", + "lens": "42793535312de4e3e3df16a69cb85f5df3b14f72", + "lens-ui-telemetry": "d6c4e330d170eefc6214dbf77a53de913fa3eebc", + "map": "7902b2e2a550e0b73fd5aa6c4e2ba3a4e6558877", + "maps-telemetry": "1b2e468ec4dd1207e417b98f84b24cd87e1efd14", + "metrics-explorer-view": "713dbf1ab5e067791d19170f715eb82cf07ebbcc", + "ml-job": "12e21f1b1adfcc1052dc0b10c7459de875653b94", + "ml-module": "c88b6a012cfb7b7adb7629b1edeab6b83f1fd048", + "ml-trained-model": "49a1685d79990ad05ea1d1d30e28456fe002f3b9", + "monitoring-telemetry": "24f7393dfacb6c7b0f7ad7d242171a1c29feaa48", + "osquery-manager-usage-metric": "23a8f08a98dd0f58ab4e559daa35b06edc40ed4f", + "osquery-pack": "407f82b8f05f02a04627993e6d2a071b4d71f816", + "osquery-pack-asset": "e10ca4b6ac5dff1954b5140ed97c3187d353a029", + "osquery-saved-query": "86bd3e1d39c470375263b28437c67b4f96667bc2", + "query": "495b96d251383d44b48ad3ccd831b857f909ae41", + "rules-settings": "9854495c3b54b16a6625fb250c35e5504da72266", + "sample-data-telemetry": "c38daf1a49ed24f2a4fb091e6e1e833fccf19935", + "search": "928589cf4b2497623faa48d22f406bab4721704c", + "search-session": "e0f495ac1b3bb40ba3eb3aea90b2d11e6c8d4c32", + "search-telemetry": "ab67ef721f294f28d5e10febbd20653347720188", + "security-rule": "1ff82dfb2298c3caf6888fc3ef15c6bf7a628877", + "security-solution-signals-migration": "c2db409c1857d330beb3d6fd188fa186f920302c", + "siem-detection-engine-rule-actions": "dbf3747aad2b986534b052703063a7026b25f485", + "siem-ui-timeline": "e9d6b3a9fd7af6dc502293c21cbdb309409f3996", + "siem-ui-timeline-note": "13c9d4c142f96624a93a623c6d7cba7e1ae9b5a6", + "siem-ui-timeline-pinned-event": "96a43d59b9e2fc11f12255a0cb47ef0a3d83af4c", + "space": "9542afcd6fd71558623c09151e453c5e84b4e5e1", + "spaces-usage-stats": "084bd0f080f94fb5735d7f3cf12f13ec92f36bad", + "synthetics-monitor": "5d0a69fac9d6cfdacfa1962274344aecb596167a", + "synthetics-param": "9776c9b571d35f0d0397e8915e035ea1dc026db7", + "synthetics-privates-locations": "7d032fc788905e32152029ae7ab3d6038c48ae44", + "tag": "87f21f07df9cc37001b15a26e413c18f50d1fbfe", + "task": "ebcc113df12f14bf627dbd335ba78507187b48a3", + "telemetry": "561b329aaed3c15b91aaf2075645be3097247612", + "ui-metric": "410a8ad28e0f44b161c960ff0ce950c712b17c52", + "upgrade-assistant-ml-upgrade-operation": "e20ff1efa3c4757f5e7ff5fb897c557b08524c3a", + "upgrade-assistant-reindex-operation": "c7442ffe34954c117a74bf138e48e4c25095a6cf", + "upgrade-assistant-telemetry": "12bcbfc4e4ce64d2ca7c24f9acccd331a2bd2ab6", + "uptime-dynamic-settings": "9a63ce80904a04be114749e426882dc3ff011137", + "uptime-synthetics-api-key": "599319bedbfa287e8761e1ba49d536417a33fa13", + "url": "2422b3cbe0af71f7a9c2e228e19a972e759c56d4", + "usage-counters": "f478b2668be350f5bdc08d9e1cf6fbce0e079f61", + "visualization": "3aff13fbc2223de74167be6a78537812c8b9d236", + "workplace_search_telemetry": "10e278fe9ae1396bfc36ae574bc387d7e696d43f", } `); }); From cd3ae20e2f2e4c485d25e00e13145dc0b4dd5fec Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Feb 2023 08:32:30 +0100 Subject: [PATCH 04/13] updating snapshots again --- .../saved_objects/migrations/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index 26b27ee25fc715..d3913a9c2361cc 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -104,7 +104,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "deff33c5b5150296c79e52dd1f4905ef189e7517", "ingest-download-sources": "95a15b6589ef46e75aca8f7e534c493f99cc3ccd", "ingest-outputs": "f5adeb3f6abc732a6067137e170578dbf1f58c62", - "ingest-package-policies": "af65ee5c419bcf8ecc05ec2d74b3662f0fb30d21", + "ingest-package-policies": "f20a00d14ba0651e0dbdada40f9762926819e298", "ingest_manager_settings": "fb75bff08a8de3435b23664b1191f9244a255701", "inventory-view": "6d47ef0b38166ecbd1c2fc7394599a4500db1ae4", "kql-telemetry": "23ed96ff02cd69cbfaa22f313cae3a54c434db51", From 70bdfb6b88f9109ecbf4ea638cb11483ecee023c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Feb 2023 08:54:16 +0100 Subject: [PATCH 05/13] add TS doc --- .../core-saved-objects-server/index.ts | 1 + .../src/model_version/index.ts | 1 + .../src/model_version/migration.ts | 47 ++++++++++++++++--- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index ebc933224b2c54..96f45e3a9a9cd7 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -99,6 +99,7 @@ export type { SavedObjectModelMigrationContext, SavedObjectModelMigrationFn, SavedObjectModelBidirectionalMigration, + SavedObjectModelMigrationResult, } from './src/model_version'; export type { diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts index feb2feeb0e67e9..9624473a8b70a6 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts @@ -11,6 +11,7 @@ export type { SavedObjectModelMigrationContext, SavedObjectModelMigrationFn, SavedObjectModelBidirectionalMigration, + SavedObjectModelMigrationResult, } from './migration'; export type { diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts index 9ecee81c18594b..9fff486146f7f1 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts @@ -9,14 +9,18 @@ import type { SavedObjectSanitizedDoc } from '../serialization'; import type { SavedObjectsMigrationLogger } from '../migration'; -// alias to more easily adapt later +/** + * Document type used during model migration. + * + * @public + */ export type SavedObjectModelMigrationDoc = SavedObjectSanitizedDoc; -export type SavedObjectModelMigrationFn = ( - doc: SavedObjectModelMigrationDoc, - context: SavedObjectModelMigrationContext -) => SavedObjectModelMigrationDoc; - +/** + * Context passed down to {@link SavedObjectModelMigrationFn | migration functions}. + * + * @public + */ export interface SavedObjectModelMigrationContext { /** * logger instance to be used by the migration handler @@ -28,6 +32,37 @@ export interface SavedObjectModelMigrationContext { readonly modelVersion: number; } +/** + * Return type for the {@link SavedObjectModelMigrationFn | migration functions} + * + * @public + */ +export interface SavedObjectModelMigrationResult { + document: SavedObjectModelMigrationDoc; +} + +/** + * Migration function for the model version API. + * + * Similar to the old migration system, model version migrations take the document to migrate + * and a context object as input and must return the transformed document in its return value. + * + * @public + */ +export type SavedObjectModelMigrationFn = ( + document: SavedObjectModelMigrationDoc, + context: SavedObjectModelMigrationContext +) => SavedObjectModelMigrationResult; + +/** + * A bidirectional migration. + * + * Bidirectional migrations define migration functions that can be used to + * migrate from the lower version to the higher one (`up`), and the other way around, + * from the higher version to the lower one (`down`) + * + * @public + */ export interface SavedObjectModelBidirectionalMigration< PreviousAttributes = unknown, NewAttributes = unknown From e37f874964ae3b758dec33a7a05680c1136e426c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Feb 2023 09:15:52 +0100 Subject: [PATCH 06/13] add doc for bimig --- .../src/model_version/migration.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts index 9fff486146f7f1..1df809d8582089 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts @@ -67,6 +67,12 @@ export interface SavedObjectModelBidirectionalMigration< PreviousAttributes = unknown, NewAttributes = unknown > { + /** + * The upward (old=>new) migration. + */ up: SavedObjectModelMigrationFn; + /** + * The downward (new=>old) migration. + */ down: SavedObjectModelMigrationFn; } From bd483319dc9d92dc33bf66d97345380bf4494a47 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 08:26:25 +0100 Subject: [PATCH 07/13] add deprecatedMappings to model change --- .../src/model_version/migration.ts | 4 ++-- .../src/model_version/model_version.ts | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts index 1df809d8582089..e64d3e7b165e08 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts @@ -68,11 +68,11 @@ export interface SavedObjectModelBidirectionalMigration< NewAttributes = unknown > { /** - * The upward (old=>new) migration. + * The upward (previous=>next) migration. */ up: SavedObjectModelMigrationFn; /** - * The downward (new=>old) migration. + * The downward (next=>previous) migration. */ down: SavedObjectModelMigrationFn; } diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts index a1f40963843e07..50b5458eb862c6 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -45,14 +45,19 @@ export interface SavedObjectsModelExpansionChange< * The type of {@link SavedObjectsModelChange | change}, used to identify them internally. */ type: 'expansion'; + /** + * (optional) A bidirectional migration to migrate the data from and/or to the previous model version. + */ + migration?: SavedObjectModelBidirectionalMigration; /** * (optional) The new mappings introduced in this version. */ addedMappings?: SavedObjectsMappingProperties; /** - * (optional) A bidirectional migration to migrate the data from and/or to the previous model version. + * (optional) A list of paths to mappings to flag as deprecated. Deprecated mappings should no longer be used and will + * eventually be deleted later. */ - migration?: SavedObjectModelBidirectionalMigration; + deprecatedMappings?: string[]; } /** From b7c34219ac2933eb90c07cede7233e962331f4c3 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 08:40:23 +0100 Subject: [PATCH 08/13] rename attribute --- .../core-saved-objects-server/src/saved_objects_type.ts | 2 +- .../src/extract_migration_info.test.ts | 6 +++--- .../src/extract_migration_info.ts | 4 ++-- .../src/get_migration_hash.ts | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts index 370dbd8acb2186..b9fa8e63651144 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts @@ -227,7 +227,7 @@ export interface SavedObjectsType { * @remarks All types will be forced to switch to use the new API in a later version. This switch is * allowing types owners to switch their types before the milestone. */ - switchToModelVersionAfter?: string; + switchToModelVersionAt?: string; } /** diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts index 65853ffc9494b0..a7d0f7e1da3828 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts @@ -160,13 +160,13 @@ describe('extractMigrationInfo', () => { }); describe('modelVersions', () => { - it('returns the correct switchToModelVersionAfter', () => { + it('returns the correct switchToModelVersionAt', () => { const type = createType({ - switchToModelVersionAfter: '8.8.0', + switchToModelVersionAt: '8.8.0', }); const output = extractMigrationInfo(type); - expect(output.switchToModelVersionAfter).toEqual('8.8.0'); + expect(output.switchToModelVersionAt).toEqual('8.8.0'); }); it('returns a proper summary of the model versions', () => { diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts index 58779f0aace082..756de2d1c1f1c8 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts @@ -21,7 +21,7 @@ export interface SavedObjectTypeMigrationInfo { mappings: Record; hasExcludeOnUpgrade: boolean; modelVersions: ModelVersionSummary[]; - switchToModelVersionAfter?: string; + switchToModelVersionAt?: string; } export interface ModelVersionSummary { @@ -71,6 +71,6 @@ export const extractMigrationInfo = (soType: SavedObjectsType): SavedObjectTypeM mappings: getFlattenedObject(soType.mappings ?? {}), hasExcludeOnUpgrade: !!soType.excludeOnUpgrade, modelVersions, - switchToModelVersionAfter: soType.switchToModelVersionAfter, + switchToModelVersionAt: soType.switchToModelVersionAt, }; }; diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts index b0123a39bc873c..bc6155f6bbc76b 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.ts @@ -26,7 +26,7 @@ export const getMigrationHash = (soType: SavedObjectsType): SavedObjectTypeMigra migInfo.migrationVersions.join(','), migInfo.schemaVersions.join(','), JSON.stringify(migInfo.mappings, Object.keys(migInfo.mappings).sort()), - migInfo.switchToModelVersionAfter ?? 'none', + migInfo.switchToModelVersionAt ?? 'none', migInfo.modelVersions.map(serializeModelVersion).join(','), ]; const hashFeed = hashParts.join('-'); From 5c556bd45d4092971ee72fef95cca1512eeff4c3 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 08:44:37 +0100 Subject: [PATCH 09/13] add test --- .../src/extract_migration_info.test.ts | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts index a7d0f7e1da3828..b41e403ae154a9 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts @@ -262,4 +262,61 @@ describe('extractMigrationInfo', () => { expect(output.modelVersions).toEqual([]); }); }); + + describe('migrations and modelVersions', () => { + it('generate properties for both', () => { + const type = createType({ + migrations: { + '8.3.3': dummyMigration, + '7.17.7': dummyMigration, + '8.0.2': dummyMigration, + }, + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + migration: { + up: jest.fn(), + down: jest.fn(), + }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { + foo: { + type: 'boolean', + }, + }, + }, + }, + }, + switchToModelVersionAt: '8.8.0', + }); + + const output = extractMigrationInfo(type); + + expect(output).toEqual( + expect.objectContaining({ + migrationVersions: ['7.17.7', '8.0.2', '8.3.3'], + switchToModelVersionAt: '8.8.0', + modelVersions: [ + { + version: '1', + changeType: 'expansion', + hasMigration: true, + newMappings: [], + }, + { + version: '2', + changeType: 'expansion', + newMappings: ['foo.type'], + hasMigration: false, + }, + ], + }) + ); + }); + }); }); From 59d8bab4945a2921aa673380a2bf21c4ac1f97b8 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 15:33:40 +0100 Subject: [PATCH 10/13] change migrations to transformations --- .../core-saved-objects-server/index.ts | 10 +-- .../src/model_version/index.ts | 12 +-- .../src/model_version/migration.ts | 78 ------------------ .../src/model_version/model_version.ts | 6 +- .../src/model_version/transformations.ts | 81 +++++++++++++++++++ 5 files changed, 95 insertions(+), 92 deletions(-) delete mode 100644 packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts create mode 100644 packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index 96f45e3a9a9cd7..2e6d3837cfac65 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -95,11 +95,11 @@ export type { SavedObjectsModelVersionMapProvider, SavedObjectsModelChange, SavedObjectsModelExpansionChange, - SavedObjectModelMigrationDoc, - SavedObjectModelMigrationContext, - SavedObjectModelMigrationFn, - SavedObjectModelBidirectionalMigration, - SavedObjectModelMigrationResult, + SavedObjectModelTransformationDoc, + SavedObjectModelTransformationContext, + SavedObjectModelTransformationFn, + SavedObjectModelBidirectionalTransformation, + SavedObjectModelTransformationResult, } from './src/model_version'; export type { diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts index 9624473a8b70a6..ec35e07e4f6745 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/index.ts @@ -7,12 +7,12 @@ */ export type { - SavedObjectModelMigrationDoc, - SavedObjectModelMigrationContext, - SavedObjectModelMigrationFn, - SavedObjectModelBidirectionalMigration, - SavedObjectModelMigrationResult, -} from './migration'; + SavedObjectModelTransformationDoc, + SavedObjectModelTransformationContext, + SavedObjectModelTransformationFn, + SavedObjectModelBidirectionalTransformation, + SavedObjectModelTransformationResult, +} from './transformations'; export type { SavedObjectsModelChange, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts deleted file mode 100644 index e64d3e7b165e08..00000000000000 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/migration.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { SavedObjectSanitizedDoc } from '../serialization'; -import type { SavedObjectsMigrationLogger } from '../migration'; - -/** - * Document type used during model migration. - * - * @public - */ -export type SavedObjectModelMigrationDoc = SavedObjectSanitizedDoc; - -/** - * Context passed down to {@link SavedObjectModelMigrationFn | migration functions}. - * - * @public - */ -export interface SavedObjectModelMigrationContext { - /** - * logger instance to be used by the migration handler - */ - readonly log: SavedObjectsMigrationLogger; - /** - * The model version this migration is registered for - */ - readonly modelVersion: number; -} - -/** - * Return type for the {@link SavedObjectModelMigrationFn | migration functions} - * - * @public - */ -export interface SavedObjectModelMigrationResult { - document: SavedObjectModelMigrationDoc; -} - -/** - * Migration function for the model version API. - * - * Similar to the old migration system, model version migrations take the document to migrate - * and a context object as input and must return the transformed document in its return value. - * - * @public - */ -export type SavedObjectModelMigrationFn = ( - document: SavedObjectModelMigrationDoc, - context: SavedObjectModelMigrationContext -) => SavedObjectModelMigrationResult; - -/** - * A bidirectional migration. - * - * Bidirectional migrations define migration functions that can be used to - * migrate from the lower version to the higher one (`up`), and the other way around, - * from the higher version to the lower one (`down`) - * - * @public - */ -export interface SavedObjectModelBidirectionalMigration< - PreviousAttributes = unknown, - NewAttributes = unknown -> { - /** - * The upward (previous=>next) migration. - */ - up: SavedObjectModelMigrationFn; - /** - * The downward (next=>previous) migration. - */ - down: SavedObjectModelMigrationFn; -} diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts index 50b5458eb862c6..3094a1ccb3e09f 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -7,7 +7,7 @@ */ import type { SavedObjectsMappingProperties } from '../mapping_definition'; -import type { SavedObjectModelBidirectionalMigration } from './migration'; +import type { SavedObjectModelBidirectionalTransformation } from './transformations'; /** * Represents a model version of a given savedObjects type. @@ -46,9 +46,9 @@ export interface SavedObjectsModelExpansionChange< */ type: 'expansion'; /** - * (optional) A bidirectional migration to migrate the data from and/or to the previous model version. + * (optional) A bidirectional transformation to migrate the document from and/or to the previous model version. */ - migration?: SavedObjectModelBidirectionalMigration; + transformation?: SavedObjectModelBidirectionalTransformation; /** * (optional) The new mappings introduced in this version. */ diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts new file mode 100644 index 00000000000000..5af4866ddd5767 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectSanitizedDoc } from '../serialization'; +import type { SavedObjectsMigrationLogger } from '../migration'; + +/** + * Document type used during model migration. + * + * @public + */ +export type SavedObjectModelTransformationDoc = SavedObjectSanitizedDoc; + +/** + * Context passed down to {@link SavedObjectModelTransformationFn | transformation functions}. + * + * @public + */ +export interface SavedObjectModelTransformationContext { + /** + * logger instance to be used by the migration handler + */ + readonly log: SavedObjectsMigrationLogger; + /** + * The model version this migration is registered for + */ + readonly modelVersion: number; +} + +/** + * Return type for the {@link SavedObjectModelTransformationFn | transformation functions} + * + * @public + */ +export interface SavedObjectModelTransformationResult { + document: SavedObjectModelTransformationDoc; +} + +/** + * Transformation function for the model version API. + * + * Similar to the old migration system, model version transformations take the document to migrate + * and a context object as input and must return the transformed document in its return value. + * + * @public + */ +export type SavedObjectModelTransformationFn< + InputAttributes = unknown, + OutputAttributes = unknown +> = ( + document: SavedObjectModelTransformationDoc, + context: SavedObjectModelTransformationContext +) => SavedObjectModelTransformationResult; + +/** + * A bidirectional transformation. + * + * Bidirectional transformations define migration functions that can be used to + * transform a document from the lower version to the higher one (`up`), and + * the other way around, from the higher version to the lower one (`down`) + * + * @public + */ +export interface SavedObjectModelBidirectionalTransformation< + PreviousAttributes = unknown, + NewAttributes = unknown +> { + /** + * The upward (previous=>next) transformation. + */ + up: SavedObjectModelTransformationFn; + /** + * The downward (next=>previous) transformation. + */ + down: SavedObjectModelTransformationFn; +} From 131fab83cd18ad87049198cf73c7a5b7a9b8f73c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 15:42:57 +0100 Subject: [PATCH 11/13] update snapshot --- .../saved_objects/migrations/check_registered_types.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index d3913a9c2361cc..bafc64776b6dd4 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -74,8 +74,8 @@ describe('checking migration metadata changes on all registered SO types', () => "cases-configure": "25099c9e4bbb91e01e334848c605b4a5de5c9fce", "cases-connector-mappings": "8de3b77dc6ae8ee62cce2b58a222471dfc3dbdad", "cases-telemetry": "fdeddcef28c75d8c66422475a829e75d37f0b668", - "cases-user-actions": "ca0dbab8f8f169ef66f92570b45ec02e57a82d78", - "config": "0090bed77e173b5d2e4b13a60388c153910cd967", + "cases-user-actions": "cfd388d2ca27b3abfd3955dc41428fb229989921", + "config": "97e16b8f5dc10c404fd3b201ef36bc6c3c63dc80", "config-global": "d9791e8f73edee884630e1cb6e4954ae513ce75e", "connector_token": "fb05ff5afdcb6e2f20c9c6513ff7a1ab12b66f36", "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", @@ -101,7 +101,7 @@ describe('checking migration metadata changes on all registered SO types', () => "index-pattern": "cd51191712081278c2af83d16552c3438ef83353", "infrastructure-monitoring-log-view": "8040108f02ef27419cff79077384379709d44bbc", "infrastructure-ui-source": "2311f7d0abe2a713aa71e30ee24f78828d4acfc1", - "ingest-agent-policies": "deff33c5b5150296c79e52dd1f4905ef189e7517", + "ingest-agent-policies": "e5bb18f8c1d1106139e82fccb93fce01b21fde9b", "ingest-download-sources": "95a15b6589ef46e75aca8f7e534c493f99cc3ccd", "ingest-outputs": "f5adeb3f6abc732a6067137e170578dbf1f58c62", "ingest-package-policies": "f20a00d14ba0651e0dbdada40f9762926819e298", From 888636b7be486c306413251150152bbe969bc110 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 16:00:42 +0100 Subject: [PATCH 12/13] fix tests --- .../src/extract_migration_info.test.ts | 6 +++--- .../src/extract_migration_info.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts index b41e403ae154a9..c94d7fe3ceb948 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.test.ts @@ -175,7 +175,7 @@ describe('extractMigrationInfo', () => { '1': { modelChange: { type: 'expansion', - migration: { + transformation: { up: jest.fn(), down: jest.fn(), }, @@ -217,7 +217,7 @@ describe('extractMigrationInfo', () => { '1': { modelChange: { type: 'expansion', - migration: { + transformation: { up: jest.fn(), down: jest.fn(), }, @@ -275,7 +275,7 @@ describe('extractMigrationInfo', () => { '1': { modelChange: { type: 'expansion', - migration: { + transformation: { up: jest.fn(), down: jest.fn(), }, diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts index 756de2d1c1f1c8..329b8f23b4a0df 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/extract_migration_info.ts @@ -56,7 +56,7 @@ export const extractMigrationInfo = (soType: SavedObjectsType): SavedObjectTypeM return { version, changeType: entry.modelChange.type, - hasMigration: !!entry.modelChange.migration, + hasMigration: !!entry.modelChange.transformation, newMappings: Object.keys(getFlattenedObject(entry.modelChange.addedMappings ?? {})), }; }); From 790897f59988b189ecbb2caa3b8d2a0b74de91d6 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 7 Feb 2023 16:11:13 +0100 Subject: [PATCH 13/13] fix more types --- .../src/get_migration_hash.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts index f1ff5510da8fb4..ab9a6aba7b6e4d 100644 --- a/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts +++ b/packages/core/test-helpers/core-test-helpers-so-type-serializer/src/get_migration_hash.test.ts @@ -329,7 +329,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -347,7 +347,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -370,7 +370,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -396,7 +396,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, }, @@ -411,7 +411,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -429,7 +429,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -452,7 +452,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': { @@ -470,7 +470,7 @@ describe('getMigrationHash', () => { '1': { modelChange: { type: 'expansion', - migration: { up: jest.fn(), down: jest.fn() }, + transformation: { up: jest.fn(), down: jest.fn() }, }, }, '2': {