From 1bca7a346f3daab1733c7310e1d88d2edfc47b5f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 27 Apr 2023 11:23:46 +0200 Subject: [PATCH 1/7] start adapting the version utils --- .../index.ts | 10 +- .../src/mappings/index.ts | 2 + .../src/mappings/types.ts | 17 +- .../src/model_version/conversion.test.ts | 17 ++ .../src/model_version/conversion.ts | 8 + .../model_version/get_version_delta.test.ts | 58 ++--- .../src/model_version/get_version_delta.ts | 18 +- .../src/model_version/index.ts | 10 +- .../model_version_from_mappings.test.ts | 34 +-- .../model_version_from_mappings.ts | 18 +- .../src/model_version/version_compare.test.ts | 84 +++---- .../src/model_version/version_compare.ts | 22 +- .../src/model_version/version_map.test.ts | 147 +++++++++++- .../src/model_version/version_map.ts | 39 +++- .../src/zdt/context/create_context.ts | 4 +- .../src/zdt/context/types.ts | 6 +- .../zdt/utils/check_version_compatibility.ts | 12 +- .../utils/generate_additive_mapping_diff.ts | 8 +- .../utils/outdated_documents_query.test.ts | 217 ++++++++++++------ .../src/zdt/utils/outdated_documents_query.ts | 34 +-- 20 files changed, 527 insertions(+), 238 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 981783bb05fd5..759ede0401feb 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -49,13 +49,17 @@ export { modelVersionToVirtualVersion, getModelVersionMapForTypes, getLatestModelVersion, + getCurrentVirtualVersion, + getLatestMigrationVersion, + getVirtualVersionMap, type ModelVersionMap, - compareModelVersions, + type VirtualVersionMap, + compareVirtualVersions, type CompareModelVersionMapParams, type CompareModelVersionStatus, type CompareModelVersionDetails, type CompareModelVersionResult, - getModelVersionsFromMappings, - getModelVersionsFromMappingMeta, + getVirtualVersionsFromMappings, + getVirtualVersionsFromMappingMeta, getModelVersionDelta, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index ee51fa9aaf4bb..779ff28fb077c 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -9,6 +9,8 @@ export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib'; export type { SavedObjectsTypeMappingDefinitions, + V2AlgoIndexMappingMeta, + ZdtAlgoIndexMappingMeta, IndexMappingMeta, IndexMapping, IndexTypesMap, diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index c756f0534db67..a8c74646de1b1 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -59,7 +59,7 @@ export interface IndexMapping { export type IndexTypesMap = Record; /** @internal */ -export interface IndexMappingMeta { +export interface V2AlgoIndexMappingMeta { /** * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') * with each key being a root-level mapping property, and each value being @@ -74,18 +74,22 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the v2 migration algorithm. */ indexTypesMap?: IndexTypesMap; +} + +/** @internal */ +export interface ZdtAlgoIndexMappingMeta { /** - * The current model versions of the mapping of the index. + * The current virtual version of the mapping of the index. * * @remark: Only defined for indices using the zdt migration algorithm. */ - mappingVersions?: { [k: string]: number }; + mappingVersions?: { [k: string]: string }; /** - * The current model versions of the documents of the index. + * The current virtual versions of the documents of the index. * * @remark: Only defined for indices using the zdt migration algorithm. */ - docVersions?: { [k: string]: number }; + docVersions?: { [k: string]: string }; /** * Info about the current state of the migration. * Should only be present if a migration is in progress or was interrupted. @@ -95,6 +99,9 @@ export interface IndexMappingMeta { migrationState?: IndexMappingMigrationStateMeta; } +/** @internal */ +export type IndexMappingMeta = V2AlgoIndexMappingMeta & ZdtAlgoIndexMappingMeta; + /** @internal */ export interface IndexMappingMigrationStateMeta { /** diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts index b65bdbce41b7a..3c89894058f57 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts @@ -11,6 +11,7 @@ import { virtualVersionToModelVersion, modelVersionToVirtualVersion, assertValidModelVersion, + assertValidVirtualVersion, } from './conversion'; describe('isVirtualModelVersion', () => { @@ -103,3 +104,19 @@ describe('assertValidModelVersion', () => { expect(assertValidModelVersion('3')).toEqual(3); }); }); + +describe('assertValidVirtualVersion', () => { + it('throws if the provided value is not a valid semver', () => { + expect(() => assertValidVirtualVersion('foooo')).toThrowErrorMatchingInlineSnapshot( + `"Virtual versions must be valid semver versions"` + ); + expect(() => assertValidVirtualVersion('1.2.3.4.5.6.7')).toThrowErrorMatchingInlineSnapshot( + `"Virtual versions must be valid semver versions"` + ); + }); + + it('returns the virtual version', () => { + expect(assertValidVirtualVersion('7.17.5')).toEqual('7.17.5'); + expect(assertValidVirtualVersion('10.3.0')).toEqual('10.3.0'); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts index c3765ca3c9be9..b10b85084eca5 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts @@ -89,6 +89,14 @@ export const assertValidModelVersion = (modelVersion: string | number): number = return modelVersion; }; +export const assertValidVirtualVersion = (virtualVersion: string): string => { + const semver = Semver.parse(virtualVersion); + if (!semver) { + throw new Error('Virtual versions must be valid semver versions'); + } + return virtualVersion; +}; + const _isVirtualModelVersion = (semver: Semver.SemVer): boolean => { return semver.major === modelVersionVirtualMajor && semver.patch === 0; }; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts index 521ddd2a0efc3..4c1bbeaf7f940 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts @@ -12,12 +12,12 @@ describe('getModelVersionDelta', () => { it('generates an upward delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 1, - b: 1, + a: '10.1.0', + b: '10.1.0', }, targetVersions: { - a: 2, - b: 3, + a: '10.2.0', + b: '10.3.0', }, deletedTypes: [], }); @@ -26,13 +26,13 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 1, - target: 2, + current: '10.1.0', + target: '10.2.0', }, { name: 'b', - current: 1, - target: 3, + current: '10.1.0', + target: '10.3.0', }, ]); }); @@ -40,12 +40,12 @@ describe('getModelVersionDelta', () => { it('generates a downward delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '10.2.0', }, targetVersions: { - a: 1, - b: 1, + a: '10.1.0', + b: '7.17.2', }, deletedTypes: [], }); @@ -54,13 +54,13 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 4, - target: 1, + current: '10.4.0', + target: '10.1.0', }, { name: 'b', - current: 2, - target: 1, + current: '10.2.0', + target: '7.17.2', }, ]); }); @@ -68,12 +68,12 @@ describe('getModelVersionDelta', () => { it('generates a noop delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '8.9.2', }, targetVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '8.9.2', }, deletedTypes: [], }); @@ -85,11 +85,11 @@ describe('getModelVersionDelta', () => { it('ignores deleted types', () => { const result = getModelVersionDelta({ currentVersions: { - a: 1, - b: 3, + a: '10.1.0', + b: '10.3.0', }, targetVersions: { - a: 2, + a: '10.2.0', }, deletedTypes: ['b'], }); @@ -98,8 +98,8 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 1, - target: 2, + current: '10.1.0', + target: '10.2.0', }, ]); }); @@ -108,12 +108,12 @@ describe('getModelVersionDelta', () => { expect(() => getModelVersionDelta({ currentVersions: { - a: 1, - b: 2, + a: '10.1.0', + b: '10.2.0', }, targetVersions: { - a: 2, - b: 1, + a: '10.2.0', + b: '10.1.0', }, deletedTypes: [], }) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts index f39c52b47f9f7..2426bc4a5770f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -6,12 +6,12 @@ * Side Public License, v 1. */ -import type { ModelVersionMap } from './version_map'; -import { compareModelVersions } from './version_compare'; +import type { VirtualVersionMap, VirtualVersion } from './version_map'; +import { compareVirtualVersions } from './version_compare'; interface GetModelVersionDeltaOpts { - currentVersions: ModelVersionMap; - targetVersions: ModelVersionMap; + currentVersions: VirtualVersionMap; + targetVersions: VirtualVersionMap; deletedTypes: string[]; } @@ -26,9 +26,9 @@ interface ModelVersionDeltaTypeResult { /** the name of the type */ name: string; /** the current version the type is at */ - current: number; + current: VirtualVersion; /** the target version the type should go to */ - target: number; + target: VirtualVersion; } /** @@ -41,7 +41,7 @@ export const getModelVersionDelta = ({ targetVersions, deletedTypes, }: GetModelVersionDeltaOpts): ModelVersionDeltaResult => { - const compared = compareModelVersions({ + const compared = compareVirtualVersions({ indexVersions: currentVersions, appVersions: targetVersions, deletedTypes, @@ -78,8 +78,8 @@ const getTypeDelta = ({ targetVersions, }: { type: string; - currentVersions: ModelVersionMap; - targetVersions: ModelVersionMap; + currentVersions: VirtualVersionMap; + targetVersions: VirtualVersionMap; }): ModelVersionDeltaTypeResult => { const currentVersion = currentVersions[type]; const targetVersion = targetVersions[type]; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 2179199921a82..7279470d26b8f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -16,17 +16,21 @@ export { export { getModelVersionMapForTypes, getLatestModelVersion, + getCurrentVirtualVersion, + getVirtualVersionMap, + getLatestMigrationVersion, type ModelVersionMap, + type VirtualVersionMap, } from './version_map'; export { - compareModelVersions, + compareVirtualVersions, type CompareModelVersionMapParams, type CompareModelVersionStatus, type CompareModelVersionDetails, type CompareModelVersionResult, } from './version_compare'; export { - getModelVersionsFromMappings, - getModelVersionsFromMappingMeta, + getVirtualVersionsFromMappings, + getVirtualVersionsFromMappingMeta, } from './model_version_from_mappings'; export { getModelVersionDelta } from './get_version_delta'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts index 8fea10f11f6b1..ec8d36a53fa9d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts @@ -7,7 +7,7 @@ */ import type { IndexMapping, IndexMappingMeta } from '../mappings'; -import { getModelVersionsFromMappings } from './model_version_from_mappings'; +import { getVirtualVersionsFromMappings } from './model_version_from_mappings'; describe('getModelVersionsFromMappings', () => { const createIndexMapping = (parts: Partial = {}): IndexMapping => ({ @@ -20,41 +20,41 @@ describe('getModelVersionsFromMappings', () => { it('retrieves the version map from docVersions', () => { const mappings = createIndexMapping({ docVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '8.16.2', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'docVersions' }); expect(versionMap).toEqual({ - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '8.16.2', }); }); it('retrieves the version map from mappingVersions', () => { const mappings = createIndexMapping({ mappingVersions: { - foo: 2, - bar: 7, + foo: '10.2.0', + bar: '7.17.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'mappingVersions' }); expect(versionMap).toEqual({ - foo: 2, - bar: 7, + foo: '10.2.0', + bar: '7.17.0', }); }); it('returns undefined for docVersions if meta field is not present', () => { const mappings = createIndexMapping({ mappingVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '10.5.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'docVersions' }); expect(versionMap).toBeUndefined(); }); @@ -62,11 +62,11 @@ describe('getModelVersionsFromMappings', () => { it('returns undefined for mappingVersions if meta field is not present', () => { const mappings = createIndexMapping({ docVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '10.5.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'mappingVersions' }); expect(versionMap).toBeUndefined(); }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index 01fc57d46462b..8dc28969d8476 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -7,8 +7,8 @@ */ import type { IndexMapping, IndexMappingMeta } from '../mappings'; -import type { ModelVersionMap } from './version_map'; -import { assertValidModelVersion } from './conversion'; +import type { VirtualVersionMap } from './version_map'; +import { assertValidVirtualVersion } from './conversion'; export interface GetModelVersionsFromMappingsOpts { mappings: IndexMapping; @@ -20,16 +20,16 @@ export interface GetModelVersionsFromMappingsOpts { /** * Build the version map from the specified source of the provided mappings. */ -export const getModelVersionsFromMappings = ({ +export const getVirtualVersionsFromMappings = ({ mappings, source, knownTypes, -}: GetModelVersionsFromMappingsOpts): ModelVersionMap | undefined => { +}: GetModelVersionsFromMappingsOpts): VirtualVersionMap | undefined => { if (!mappings._meta) { return undefined; } - return getModelVersionsFromMappingMeta({ + return getVirtualVersionsFromMappingMeta({ meta: mappings._meta, source, knownTypes, @@ -46,20 +46,20 @@ export interface GetModelVersionsFromMappingMetaOpts { /** * Build the version map from the specified source of the provided mappings meta. */ -export const getModelVersionsFromMappingMeta = ({ +export const getVirtualVersionsFromMappingMeta = ({ meta, source, knownTypes, -}: GetModelVersionsFromMappingMetaOpts): ModelVersionMap | undefined => { +}: GetModelVersionsFromMappingMetaOpts): VirtualVersionMap | undefined => { const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; if (!indexVersions) { return undefined; } const typeSet = knownTypes ? new Set(knownTypes) : undefined; - return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { if (!typeSet || typeSet.has(type)) { - map[type] = assertValidModelVersion(rawVersion); + map[type] = assertValidVirtualVersion(rawVersion); } return map; }, {}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts index eba6fe1837cce..64feb07455eb3 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts @@ -6,18 +6,18 @@ * Side Public License, v 1. */ -import { compareModelVersions } from './version_compare'; +import { compareVirtualVersions } from './version_compare'; describe('compareModelVersions', () => { it('returns the correct value for greater app version', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 3, - bar: 2, + foo: '10.3.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -26,14 +26,14 @@ describe('compareModelVersions', () => { }); it('returns the correct value for lesser app version', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 1, - bar: 2, + foo: '10.1.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -42,14 +42,14 @@ describe('compareModelVersions', () => { }); it('returns the correct value for equal versions', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -58,13 +58,13 @@ describe('compareModelVersions', () => { }); it('handles new types not being present in the index', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 2, - new: 1, + foo: '10.2.0', + bar: '10.1.0', }, indexVersions: { - foo: 2, + foo: '10.2.0', }, deletedTypes: [], }); @@ -73,13 +73,13 @@ describe('compareModelVersions', () => { }); it('handles types not being present in the app', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 3, + foo: '10.3.0', }, indexVersions: { - foo: 2, - old: 1, + foo: '10.2.0', + old: '10.1.0', }, deletedTypes: [], }); @@ -88,16 +88,16 @@ describe('compareModelVersions', () => { }); it('returns the correct value for conflicts', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, - b: 3, - c: 3, + a: '10.3.0', + b: '10.3.0', + c: '10.3.0', }, indexVersions: { - a: 2, - b: 3, - c: 4, + a: '10.2.0', + b: '10.3.0', + c: '10.4.0', }, deletedTypes: [], }); @@ -106,16 +106,16 @@ describe('compareModelVersions', () => { }); it('properly lists the details', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, - b: 3, - c: 3, + a: '10.3.0', + b: '10.3.0', + c: '10.3.0', }, indexVersions: { - a: 2, - b: 3, - c: 4, + a: '10.2.0', + b: '10.3.0', + c: '10.4.0', }, deletedTypes: [], }); @@ -126,13 +126,13 @@ describe('compareModelVersions', () => { }); it('ignores deleted types when comparing', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, + a: '10.3.0', }, indexVersions: { - a: 2, - b: 3, + a: '10.2.0', + b: '10.3.0', }, deletedTypes: ['b'], }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts index 9b8d14b7fd862..6d250167cb4e2 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts @@ -6,13 +6,14 @@ * Side Public License, v 1. */ -import type { ModelVersionMap } from './version_map'; +import Semver from 'semver'; +import type { VirtualVersionMap } from './version_map'; export interface CompareModelVersionMapParams { /** The latest model version of the types registered in the application */ - appVersions: ModelVersionMap; + appVersions: VirtualVersionMap; /** The model version stored in the index */ - indexVersions: ModelVersionMap; + indexVersions: VirtualVersionMap; /** The list of deleted types to exclude during the compare process */ deletedTypes: string[]; } @@ -37,7 +38,7 @@ export interface CompareModelVersionResult { details: CompareModelVersionDetails; } -export const compareModelVersions = ({ +export const compareVirtualVersions = ({ appVersions, indexVersions, deletedTypes, @@ -53,14 +54,19 @@ export const compareModelVersions = ({ }; allTypes.forEach((type) => { - const appVersion = appVersions[type] ?? 0; - const indexVersion = indexVersions[type] ?? 0; + const appVersion = appVersions[type] ?? '0.0.0'; + const indexVersion = indexVersions[type] ?? '0.0.0'; - if (appVersion > indexVersion) { + const comparison = Semver.compare(appVersion, indexVersion); + + if (comparison > 0) { + // app version greater than index version details.greater.push(type); - } else if (appVersion < indexVersion) { + } else if (comparison < 0) { + // app version lower than index version details.lesser.push(type); } else { + // // app version equal to index version details.equal.push(type); } }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts index aafb83ab96009..1849809c516f9 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -7,7 +7,13 @@ */ import type { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { getModelVersionMapForTypes, getLatestModelVersion } from './version_map'; +import { + getModelVersionMapForTypes, + getLatestModelVersion, + getLatestMigrationVersion, + getCurrentVirtualVersion, + getVirtualVersionMap, +} from './version_map'; describe('ModelVersion map utilities', () => { const buildType = (parts: Partial = {}): SavedObjectsType => ({ @@ -24,6 +30,8 @@ describe('ModelVersion map utilities', () => { }, }); + const dummyMigration = jest.fn(); + describe('getLatestModelVersion', () => { it('returns 0 when no model versions are registered', () => { expect(getLatestModelVersion(buildType({ modelVersions: {} }))).toEqual(0); @@ -116,4 +124,141 @@ describe('ModelVersion map utilities', () => { }); }); }); + + describe('getLatestMigrationVersion', () => { + it('returns 0.0.0 when no migrations are registered', () => { + expect(getLatestMigrationVersion(buildType({ migrations: {} }))).toEqual('0.0.0'); + expect(getLatestMigrationVersion(buildType({ migrations: undefined }))).toEqual('0.0.0'); + }); + + it('throws if an invalid version is provided', () => { + expect(() => + getLatestMigrationVersion( + buildType({ + migrations: { + foo: dummyMigration, + '8.6.0': dummyMigration, + }, + }) + ) + ).toThrowError(); + }); + + it('returns the latest registered version', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + }) + ) + ).toEqual('8.6.0'); + }); + + it('accepts provider functions', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: () => ({ + '7.17.2': dummyMigration, + '8.4.0': dummyMigration, + }), + }) + ) + ).toEqual('8.4.0'); + }); + + it('supports unordered maps', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.7.0': dummyMigration, + '8.2.0': dummyMigration, + }, + }) + ) + ).toEqual('8.7.0'); + }); + }); + + describe('getCurrentVirtualVersion', () => { + it('returns the latest registered migration if switchToModelVersionAt is unset', () => { + expect( + getCurrentVirtualVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }) + ) + ).toEqual('8.6.0'); + }); + + it('returns the virtual version of the latest model version if switchToModelVersionAt is set', () => { + expect( + getCurrentVirtualVersion( + buildType({ + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }) + ) + ).toEqual('10.1.0'); + }); + }); + + describe('getVirtualVersionMap', () => { + it('returns the virtual version for each of the provided types', () => { + expect( + getVirtualVersionMap([ + buildType({ + name: 'foo', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }), + buildType({ + name: 'bar', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }), + buildType({ + name: 'dolly', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + }), + ]) + ).toEqual({ + foo: '10.1.0', + bar: '8.6.0', + dolly: '10.0.0', + }); + }); + }); }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts index dd05e64dbcbef..e4d2a86077065 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -6,10 +6,13 @@ * Side Public License, v 1. */ +import Semver from 'semver'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { assertValidModelVersion } from './conversion'; +import { assertValidModelVersion, modelVersionToVirtualVersion } from './conversion'; export type ModelVersionMap = Record; +export type VirtualVersion = string; +export type VirtualVersionMap = Record; /** * Returns the latest registered model version number for the given type. @@ -22,6 +25,14 @@ export const getLatestModelVersion = (type: SavedObjectsType): number => { }, 0); }; +export const getLatestMigrationVersion = (type: SavedObjectsType): string => { + const migrationMap = + typeof type.migrations === 'function' ? type.migrations() : type.migrations ?? {}; + return Object.keys(migrationMap).reduce((memo, current) => { + return Semver.gt(memo, current) ? memo : current; + }, '0.0.0'); +}; + /** * Build a version map for the given types. */ @@ -31,3 +42,29 @@ export const getModelVersionMapForTypes = (types: SavedObjectsType[]): ModelVers return versionMap; }, {}); }; + +/** + * Returns the current virtual version for the given type. + * If will either be the latest model version if the type + * already switched to using them (switchToModelVersionAt is set), + * or the latest migration version for the type otherwise. + */ +export const getCurrentVirtualVersion = (type: SavedObjectsType): string => { + if (type.switchToModelVersionAt) { + const modelVersion = getLatestModelVersion(type); + return modelVersionToVirtualVersion(modelVersion); + } else { + return getLatestMigrationVersion(type); + } +}; + +/** + * Returns a map of virtual model version for the given types. + * See {@link getCurrentVirtualVersion} + */ +export const getVirtualVersionMap = (types: SavedObjectsType[]): VirtualVersionMap => { + return types.reduce((versionMap, type) => { + versionMap[type.name] = getCurrentVirtualVersion(type); + return versionMap; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index c31d4bc799b3b..82234823d973a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { getModelVersionMapForTypes } from '@kbn/core-saved-objects-base-server-internal'; +import { getVirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; import { REMOVED_TYPES } from '../../core'; import type { MigrateIndexOptions } from '../migrate_index'; import type { MigratorContext } from './types'; @@ -33,7 +33,7 @@ export const createContext = ({ kibanaVersion, indexPrefix, types, - typeModelVersions: getModelVersionMapForTypes(types.map((type) => typeRegistry.getType(type)!)), + typeVirtualVersions: getVirtualVersionMap(types.map((type) => typeRegistry.getType(type)!)), elasticsearchClient, typeRegistry, serializer, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 95ca7282daf57..58887418eb35a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -12,7 +12,7 @@ import type { ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; import type { - ModelVersionMap, + VirtualVersionMap, SavedObjectsMigrationConfigType, } from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; @@ -30,8 +30,8 @@ export interface MigratorContext { readonly indexPrefix: string; /** Name of the types that are living in the index */ readonly types: string[]; - /** Model versions for the registered types */ - readonly typeModelVersions: ModelVersionMap; + /** Virtual versions for the registered types */ + readonly typeVirtualVersions: VirtualVersionMap; /** The client to use for communications with ES */ readonly elasticsearchClient: ElasticsearchClient; /** The maximum number of retries to attempt for a failing action */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts index c231645fb7993..651875e92c438 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -8,9 +8,9 @@ import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import { - getModelVersionsFromMappings, - compareModelVersions, - getModelVersionMapForTypes, + getVirtualVersionsFromMappings, + compareVirtualVersions, + getVirtualVersionMap, type IndexMapping, type CompareModelVersionResult, } from '@kbn/core-saved-objects-base-server-internal'; @@ -28,8 +28,8 @@ export const checkVersionCompatibility = ({ source, deletedTypes, }: CheckVersionCompatibilityOpts): CompareModelVersionResult => { - const appVersions = getModelVersionMapForTypes(types); - const indexVersions = getModelVersionsFromMappings({ + const appVersions = getVirtualVersionMap(types); + const indexVersions = getVirtualVersionsFromMappings({ mappings, source, knownTypes: types.map((type) => type.name), @@ -37,5 +37,5 @@ export const checkVersionCompatibility = ({ if (!indexVersions) { throw new Error(`Cannot check version: ${source} not present in the mapping meta`); } - return compareModelVersions({ appVersions, indexVersions, deletedTypes }); + return compareVirtualVersions({ appVersions, indexVersions, deletedTypes }); }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts index 400e01e999797..0ef7437b2d6c3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -12,8 +12,8 @@ import type { } from '@kbn/core-saved-objects-server'; import { IndexMappingMeta, - getModelVersionsFromMappingMeta, - getModelVersionMapForTypes, + getVirtualVersionsFromMappingMeta, + getVirtualVersionMap, getModelVersionDelta, } from '@kbn/core-saved-objects-base-server-internal'; @@ -35,8 +35,8 @@ export const generateAdditiveMappingDiff = ({ meta, deletedTypes, }: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { - const typeVersions = getModelVersionMapForTypes(types); - const mappingVersion = getModelVersionsFromMappingMeta({ + const typeVersions = getVirtualVersionMap(types); + const mappingVersion = getVirtualVersionsFromMappingMeta({ meta, source: 'mappingVersions', knownTypes: types.map((type) => type.name), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts index f39016b7d86a1..fb72061735eff 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts @@ -16,10 +16,13 @@ const dummyModelVersion: SavedObjectsModelVersion = { }, }; +const dummyMigration = jest.fn(); + describe('getOutdatedDocumentsQuery', () => { - it('generates the correct query', () => { + it('generates the correct query for types using model versions', () => { const fooType = createType({ name: 'foo', + switchToModelVersionAt: '8.8.0', modelVersions: { 1: dummyModelVersion, 2: dummyModelVersion, @@ -27,6 +30,7 @@ describe('getOutdatedDocumentsQuery', () => { }); const barType = createType({ name: 'bar', + switchToModelVersionAt: '8.8.0', modelVersions: { 1: dummyModelVersion, 2: dummyModelVersion, @@ -50,40 +54,11 @@ describe('getOutdatedDocumentsQuery', () => { "type": "foo", }, }, + ], + "must_not": Array [ Object { - "bool": Object { - "should": Array [ - Object { - "bool": Object { - "must": Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - "must_not": Object { - "term": Object { - "migrationVersion.foo": "10.2.0", - }, - }, - }, - }, - Object { - "bool": Object { - "must_not": Array [ - Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - Object { - "term": Object { - "typeMigrationVersion": "10.2.0", - }, - }, - ], - }, - }, - ], + "term": Object { + "typeMigrationVersion": "10.2.0", }, }, ], @@ -97,40 +72,148 @@ describe('getOutdatedDocumentsQuery', () => { "type": "bar", }, }, + ], + "must_not": Array [ Object { - "bool": Object { - "should": Array [ - Object { - "bool": Object { - "must": Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - "must_not": Object { - "term": Object { - "migrationVersion.bar": "10.3.0", - }, - }, - }, - }, - Object { - "bool": Object { - "must_not": Array [ - Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - Object { - "term": Object { - "typeMigrationVersion": "10.3.0", - }, - }, - ], - }, - }, - ], + "term": Object { + "typeMigrationVersion": "10.3.0", + }, + }, + ], + }, + }, + ], + }, + } + `); + }); + + it('generates the correct query for types still using old migrations', () => { + const fooType = createType({ + name: 'foo', + migrations: { + '7.17.2': dummyMigration, + '8.5.0': dummyMigration, + }, + }); + const barType = createType({ + name: 'bar', + migrations: () => ({ + '7.15.5': dummyMigration, + '8.7.2': dummyMigration, + }), + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.5.0", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.7.2", + }, + }, + ], + }, + }, + ], + }, + } + `); + }); + + it('generates the correct query for mixed types', () => { + const fooType = createType({ + name: 'foo', + migrations: { + '7.17.2': dummyMigration, + '8.5.0': dummyMigration, + }, + switchToModelVersionAt: '8.8.0', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + migrations: () => ({ + '7.15.5': dummyMigration, + '8.7.2': dummyMigration, + }), + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "10.2.0", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.7.2", }, }, ], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts index e15a2e447b554..40495654e6146 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts @@ -8,10 +8,7 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { - getModelVersionMapForTypes, - modelVersionToVirtualVersion, -} from '@kbn/core-saved-objects-base-server-internal'; +import { getVirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; interface GetOutdatedDocumentsQueryOps { types: SavedObjectsType[]; @@ -23,36 +20,15 @@ export const getOutdatedDocumentsQuery = ({ // Note: in theory, we could check the difference of model version with the index's // and narrow the search filter only on the type that have different versions. // however, it feels safer to just search for all outdated document, just in case. - const modelVersions = getModelVersionMapForTypes(types); + const virtualVersions = getVirtualVersionMap(types); return { bool: { should: types.map((type) => { - const virtualVersion = modelVersionToVirtualVersion(modelVersions[type.name]); + const virtualVersion = virtualVersions[type.name]; return { bool: { - must: [ - { term: { type: type.name } }, - { - bool: { - should: [ - { - bool: { - must: { exists: { field: 'migrationVersion' } }, - must_not: { term: { [`migrationVersion.${type.name}`]: virtualVersion } }, - }, - }, - { - bool: { - must_not: [ - { exists: { field: 'migrationVersion' } }, - { term: { typeMigrationVersion: virtualVersion } }, - ], - }, - }, - ], - }, - }, - ], + must: [{ term: { type: type.name } }], + must_not: [{ term: { typeMigrationVersion: virtualVersion } }], }, }; }), From f3f37ed4e68dc4d8769da71016e417bd05857d1e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 27 Apr 2023 12:01:27 +0200 Subject: [PATCH 2/7] fix/adapt zdt algo utils --- .../check_version_compatibility.test.mocks.ts | 12 +++---- .../utils/check_version_compatibility.test.ts | 36 +++++++++---------- .../generate_additive_mapping_diff.test.ts | 33 +++++++++-------- 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts index a0e03552ed946..8d7cbe657758c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts @@ -6,16 +6,16 @@ * Side Public License, v 1. */ -export const getModelVersionsFromMappingsMock = jest.fn(); -export const compareModelVersionsMock = jest.fn(); -export const getModelVersionMapForTypesMock = jest.fn(); +export const getVirtualVersionsFromMappingsMock = jest.fn(); +export const compareVirtualVersionsMock = jest.fn(); +export const getVirtualVersionMapMock = jest.fn(); jest.doMock('@kbn/core-saved-objects-base-server-internal', () => { const actual = jest.requireActual('@kbn/core-saved-objects-base-server-internal'); return { ...actual, - getModelVersionsFromMappings: getModelVersionsFromMappingsMock, - compareModelVersions: compareModelVersionsMock, - getModelVersionMapForTypes: getModelVersionMapForTypesMock, + getVirtualVersionsFromMappings: getVirtualVersionsFromMappingsMock, + compareVirtualVersions: compareVirtualVersionsMock, + getVirtualVersionMap: getVirtualVersionMapMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts index 8430f1f898426..528855417e90f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -7,14 +7,14 @@ */ import { - compareModelVersionsMock, - getModelVersionsFromMappingsMock, - getModelVersionMapForTypesMock, + compareVirtualVersionsMock, + getVirtualVersionMapMock, + getVirtualVersionsFromMappingsMock, } from './check_version_compatibility.test.mocks'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import type { IndexMapping, - ModelVersionMap, + VirtualVersionMap, CompareModelVersionResult, } from '@kbn/core-saved-objects-base-server-internal'; import { checkVersionCompatibility } from './check_version_compatibility'; @@ -27,9 +27,9 @@ describe('checkVersionCompatibility', () => { let mappings: IndexMapping; beforeEach(() => { - compareModelVersionsMock.mockReset().mockReturnValue({}); - getModelVersionsFromMappingsMock.mockReset().mockReturnValue({}); - getModelVersionMapForTypesMock.mockReset().mockReturnValue({ status: 'equal' }); + compareVirtualVersionsMock.mockReset().mockReturnValue({}); + getVirtualVersionMapMock.mockReset().mockReturnValue({}); + getVirtualVersionsFromMappingsMock.mockReset().mockReturnValue({ status: 'equal' }); types = [createType({ name: 'foo' }), createType({ name: 'bar' })]; @@ -46,8 +46,8 @@ describe('checkVersionCompatibility', () => { deletedTypes, }); - expect(getModelVersionMapForTypesMock).toHaveBeenCalledTimes(1); - expect(getModelVersionMapForTypesMock).toHaveBeenCalledWith(types); + expect(getVirtualVersionMapMock).toHaveBeenCalledTimes(1); + expect(getVirtualVersionMapMock).toHaveBeenCalledWith(types); }); it('calls getModelVersionsFromMappings with the correct parameters', () => { @@ -58,8 +58,8 @@ describe('checkVersionCompatibility', () => { deletedTypes, }); - expect(getModelVersionsFromMappingsMock).toHaveBeenCalledTimes(1); - expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ + expect(getVirtualVersionsFromMappingsMock).toHaveBeenCalledTimes(1); + expect(getVirtualVersionsFromMappingsMock).toHaveBeenCalledWith({ mappings, source: 'mappingVersions', knownTypes: ['foo', 'bar'], @@ -67,11 +67,11 @@ describe('checkVersionCompatibility', () => { }); it('calls compareModelVersions with the correct parameters', () => { - const appVersions: ModelVersionMap = { foo: 2, bar: 2 }; - const indexVersions: ModelVersionMap = { foo: 1, bar: 1 }; + const appVersions: VirtualVersionMap = { foo: '10.2.0', bar: '10.2.0' }; + const indexVersions: VirtualVersionMap = { foo: '10.1.0', bar: '10.1.0' }; - getModelVersionMapForTypesMock.mockReturnValue(appVersions); - getModelVersionsFromMappingsMock.mockReturnValue(indexVersions); + getVirtualVersionMapMock.mockReturnValue(appVersions); + getVirtualVersionsFromMappingsMock.mockReturnValue(indexVersions); checkVersionCompatibility({ types, @@ -80,8 +80,8 @@ describe('checkVersionCompatibility', () => { deletedTypes, }); - expect(compareModelVersionsMock).toHaveBeenCalledTimes(1); - expect(compareModelVersionsMock).toHaveBeenCalledWith({ + expect(compareVirtualVersionsMock).toHaveBeenCalledTimes(1); + expect(compareVirtualVersionsMock).toHaveBeenCalledWith({ appVersions, indexVersions, deletedTypes, @@ -97,7 +97,7 @@ describe('checkVersionCompatibility', () => { equal: [], }, }; - compareModelVersionsMock.mockReturnValue(expected); + compareVirtualVersionsMock.mockReturnValue(expected); const result = checkVersionCompatibility({ types, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts index ce0cba20f427c..9e711dd6daaf1 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import type { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; import { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; import { createType } from '../test_helpers'; @@ -13,20 +14,24 @@ import { createType } from '../test_helpers'; describe('generateAdditiveMappingDiff', () => { const deletedTypes = ['deletedType']; + const stubMigration = jest.fn(); + const stubModelVersion: SavedObjectsModelVersion = { modelChange: { type: 'expansion' } }; + const getTypes = () => { const foo = createType({ name: 'foo', + switchToModelVersionAt: '8.0.0', modelVersions: { - 1: { modelChange: { type: 'expansion' } }, - 2: { modelChange: { type: 'expansion' } }, + 1: stubModelVersion, + 2: stubModelVersion, }, mappings: { properties: { fooProp: { type: 'text' } } }, }); const bar = createType({ name: 'bar', - modelVersions: { - 1: { modelChange: { type: 'expansion' } }, - 2: { modelChange: { type: 'expansion' } }, + migrations: { + '8.0.0': stubMigration, + '8.5.0': stubMigration, }, mappings: { properties: { barProp: { type: 'text' } } }, }); @@ -39,8 +44,8 @@ describe('generateAdditiveMappingDiff', () => { const types = [foo, bar]; const meta: IndexMappingMeta = { mappingVersions: { - foo: 1, - bar: 1, + foo: '10.1.0', + bar: '7.9.0', }, }; @@ -61,8 +66,8 @@ describe('generateAdditiveMappingDiff', () => { const types = [foo, bar]; const meta: IndexMappingMeta = { mappingVersions: { - foo: 1, - bar: 2, + foo: '10.1.0', + bar: '8.5.0', }, }; @@ -82,9 +87,9 @@ describe('generateAdditiveMappingDiff', () => { const types = [foo, bar]; const meta: IndexMappingMeta = { mappingVersions: { - foo: 1, - bar: 1, - deletedType: 42, + foo: '10.1.0', + bar: '8.2.0', + deletedType: '10.2.0', }, }; @@ -105,8 +110,8 @@ describe('generateAdditiveMappingDiff', () => { const types = [foo, bar]; const meta: IndexMappingMeta = { mappingVersions: { - foo: 1, - bar: 3, + foo: '10.1.0', + bar: '10.1.0', }, }; From 9163bd4e4d32cfddc22d10c1eb3b7cf30d049686 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 27 Apr 2023 16:13:30 +0200 Subject: [PATCH 3/7] adapt more usages and tests --- .../src/zdt/actions/update_index_meta.test.ts | 4 ++-- .../src/zdt/model/stages/init.test.ts | 2 +- .../stages/update_document_model_version.test.ts | 8 ++++---- .../model/stages/update_document_model_version.ts | 2 +- .../stages/update_mapping_model_version.test.ts | 2 +- .../model/stages/update_mapping_model_version.ts | 2 +- .../src/zdt/next.test.ts | 4 ++-- .../src/zdt/next.ts | 4 ++-- .../src/zdt/test_helpers/context.ts | 6 +++--- .../src/zdt/utils/build_index_mappings.test.ts | 12 ++++++------ .../src/zdt/utils/build_index_mappings.ts | 8 ++++---- .../src/zdt/utils/update_index_meta.test.ts | 14 +++++++------- .../src/zdt/utils/update_index_meta.ts | 6 +++--- 13 files changed, 37 insertions(+), 37 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts index 6ed55ccb49ebe..d597e2927663b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts @@ -17,8 +17,8 @@ describe('updateIndexMeta', () => { const index = '.kibana_1'; const meta: IndexMappingMeta = { mappingVersions: { - foo: 1, - bar: 1, + foo: '10.1.0', + bar: '10.1.0', }, }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index d8c176af4be0a..05cb283d378a6 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -38,7 +38,7 @@ describe('Stage: init', () => { aliases: {}, mappings: { properties: {}, - _meta: { mappingVersions: { foo: 1, bar: 1 } }, + _meta: { mappingVersions: { foo: '10.1.0', bar: '10.1.0' } }, }, settings: {}, }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts index c3d3fd67422b9..3595b5fb43cd4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts @@ -49,8 +49,8 @@ describe('Stage: updateDocumentModelVersion', () => { it('updates state.currentIndexMeta when successful', () => { const state = createState({ currentIndexMeta: { - mappingVersions: { foo: 1, bar: 2 }, - docVersions: { foo: 0, bar: 0 }, + mappingVersions: { foo: '10.1.0', bar: '10.2.0' }, + docVersions: { foo: '0.0.0', bar: '0.0.0' }, migrationState: { convertingDocuments: true, }, @@ -63,8 +63,8 @@ describe('Stage: updateDocumentModelVersion', () => { const newState = updateDocumentModelVersion(state, res, context); expect(newState.currentIndexMeta).toEqual({ - mappingVersions: { foo: 1, bar: 2 }, - docVersions: { foo: 1, bar: 2 }, + mappingVersions: { foo: '10.1.0', bar: '10.2.0' }, + docVersions: { foo: '10.1.0', bar: '10.2.0' }, migrationState: { convertingDocuments: false, }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts index ea3ea142cbd19..94c0923df267d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts @@ -24,7 +24,7 @@ export const updateDocumentModelVersion: ModelStage< controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', currentIndexMeta: setMetaDocMigrationComplete({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts index b21ec69a531f0..ea053570b74e8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -42,7 +42,7 @@ describe('Stage: updateMappingModelVersion', () => { expect(newState.currentIndexMeta).toEqual( setMetaMappingMigrationComplete({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }) ); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts index 946c4a4ab1ef3..54cb5cff6d24a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -24,7 +24,7 @@ export const updateMappingModelVersion: ModelStage< controlState: state.aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', currentIndexMeta: setMetaMappingMigrationComplete({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts index d9135fff65a3e..00684ec46f85c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts @@ -85,7 +85,7 @@ describe('actions', () => { expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledTimes(1); expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledWith({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }); }); @@ -123,7 +123,7 @@ describe('actions', () => { expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledTimes(1); expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledWith({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index cb3e1b5b5ad27..2e6a785704b8a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -84,7 +84,7 @@ export const nextActionMap = (context: MigratorContext) => { index: state.currentIndex, meta: setMetaMappingMigrationComplete({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }), }), UPDATE_ALIASES: (state: UpdateAliasesState) => @@ -172,7 +172,7 @@ export const nextActionMap = (context: MigratorContext) => { index: state.currentIndex, meta: setMetaDocMigrationComplete({ meta: state.currentIndexMeta, - versions: context.typeModelVersions, + versions: context.typeVirtualVersions, }), }), UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: ( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index 0dafc36108b93..6fb5eb180d71e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -31,9 +31,9 @@ export const createContextMock = ( kibanaVersion: '8.7.0', indexPrefix: '.kibana', types: ['foo', 'bar'], - typeModelVersions: { - foo: 1, - bar: 2, + typeVirtualVersions: { + foo: '10.1.0', + bar: '10.2.0', }, documentMigrator: createDocumentMigrator(), migrationConfig: { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts index 35354001f6803..6469ebb7f060a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -69,14 +69,14 @@ describe('buildIndexMeta', () => { expect(meta).toEqual({ mappingVersions: { - foo: 2, - bar: 1, - dolly: 3, + foo: '10.2.0', + bar: '10.1.0', + dolly: '10.3.0', }, docVersions: { - foo: 2, - bar: 1, - dolly: 3, + foo: '10.2.0', + bar: '10.1.0', + dolly: '10.3.0', }, migrationState: { convertingDocuments: false, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index a75ebd4dbdc1e..2157f3798d4db 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -11,7 +11,7 @@ import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import { type IndexMapping, type IndexMappingMeta, - getModelVersionMapForTypes, + getVirtualVersionMap, } from '@kbn/core-saved-objects-base-server-internal'; import { getBaseMappings, buildTypesMappings } from '../../core'; @@ -48,11 +48,11 @@ interface BuildIndexMetaOpts { * @param types The list of all registered SO types. */ export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta => { - const modelVersions = getModelVersionMapForTypes(types); + const typeVersions = getVirtualVersionMap(types); return { - mappingVersions: modelVersions, - docVersions: modelVersions, + mappingVersions: typeVersions, + docVersions: typeVersions, migrationState: { convertingDocuments: false, }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts index 4298c6b027072..da9654400bf6f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts @@ -8,7 +8,7 @@ import type { IndexMappingMeta, - ModelVersionMap, + VirtualVersionMap, } from '@kbn/core-saved-objects-base-server-internal'; import { setMetaDocMigrationStarted, @@ -18,12 +18,12 @@ import { const getDefaultMeta = (): IndexMappingMeta => ({ mappingVersions: { - foo: 1, - bar: 1, + foo: '10.1.0', + bar: '10.1.0', }, docVersions: { - foo: 1, - bar: 1, + foo: '10.1.0', + bar: '10.1.0', }, migrationState: { convertingDocuments: false, @@ -33,7 +33,7 @@ const getDefaultMeta = (): IndexMappingMeta => ({ describe('setMetaMappingMigrationComplete', () => { it('updates the meta to set the mappingVersions', () => { const meta: IndexMappingMeta = getDefaultMeta(); - const versions: ModelVersionMap = { foo: 3, bar: 2 }; + const versions: VirtualVersionMap = { foo: '10.3.0', bar: '10.2.0' }; const updated = setMetaMappingMigrationComplete({ meta, versions }); @@ -67,7 +67,7 @@ describe('setMetaDocMigrationComplete', () => { convertingDocuments: true, }, }; - const versions: ModelVersionMap = { foo: 3, bar: 2 }; + const versions: VirtualVersionMap = { foo: '10.3.0', bar: '10.2.0' }; const updated = setMetaDocMigrationComplete({ meta, versions }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts index 73d693d4a1f1a..2d06f5adcb378 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts @@ -8,7 +8,7 @@ import type { IndexMappingMeta, - ModelVersionMap, + VirtualVersionMap, } from '@kbn/core-saved-objects-base-server-internal'; export const setMetaMappingMigrationComplete = ({ @@ -16,7 +16,7 @@ export const setMetaMappingMigrationComplete = ({ versions, }: { meta: IndexMappingMeta; - versions: ModelVersionMap; + versions: VirtualVersionMap; }): IndexMappingMeta => { return { ...meta, @@ -44,7 +44,7 @@ export const setMetaDocMigrationComplete = ({ versions, }: { meta: IndexMappingMeta; - versions: ModelVersionMap; + versions: VirtualVersionMap; }): IndexMappingMeta => { return { ...meta, From c91ce1e2a5a23a87e7ce919a685f0e1dc9d41731 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 1 May 2023 09:12:53 +0200 Subject: [PATCH 4/7] adapt integration tests --- .../zero_downtime/basic_document_migration.test.ts | 4 ++-- .../migrations/zero_downtime/create_index.test.ts | 8 ++++---- .../zero_downtime/mapping_version_conflict.test.ts | 4 ++-- .../migrations/zero_downtime/update_mappings.test.ts | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts index 418adb0a7894e..83f7fda262854 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts @@ -179,8 +179,8 @@ describe('ZDT upgrades - basic document migration', () => { ); expect(mappingMeta.docVersions).toEqual({ - sample_a: 2, - sample_b: 3, + sample_a: '10.2.0', + sample_b: '10.3.0', }); const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts index 760b0f113a8ff..bee9013544b72 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -81,12 +81,12 @@ describe('ZDT upgrades - running on a fresh cluster', () => { expect(mappingMeta).toEqual({ docVersions: { - foo: 2, - bar: 1, + foo: '10.2.0', + bar: '10.1.0', }, mappingVersions: { - foo: 2, - bar: 1, + foo: '10.2.0', + bar: '10.1.0', }, migrationState: expect.objectContaining({ convertingDocuments: false, diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts index 1674bf747f7b1..55ff37d47b080 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -106,8 +106,8 @@ describe('ZDT upgrades - mapping model version conflict', () => { expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappingMeta.mappingVersions).toEqual({ - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }); const records = await parseLogFile(logFilePath); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 9829da66a965e..8117b3b68137f 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -111,8 +111,8 @@ describe('ZDT upgrades - basic mapping update', () => { ); expect(mappingMeta.mappingVersions).toEqual({ - foo: 3, - bar: 2, + foo: '10.3.0', + bar: '10.2.0', }); const records = await parseLogFile(logFilePath); From 7eb03fac16db0ee02fb4faa73aedbc06e23a7c15 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 1 May 2023 12:04:32 +0200 Subject: [PATCH 5/7] add tests for zdt-v2-compat --- .../zdt_base.fixtures.ts} | 19 +- .../basic_document_migration.test.ts | 6 +- .../conversion_failures.test.ts | 6 +- .../create_index.test.ts | 2 +- .../document_cleanup.test.ts | 2 +- .../jest.integration.config.js | 2 +- .../mapping_version_conflict.test.ts | 7 +- .../rerun_same_version.test.ts | 2 +- .../standard_workflow.test.ts | 2 +- .../update_mappings.test.ts | 7 +- .../basic_document_migration.test.ts | 243 ++++++++++++++++++ .../zdt_v2_compat/create_index.test.ts | 104 ++++++++ .../zdt_v2_compat/jest.integration.config.js | 19 ++ .../zdt_v2_compat/update_mappings.test.ts | 136 ++++++++++ 14 files changed, 547 insertions(+), 10 deletions(-) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime/base.fixtures.ts => fixtures/zdt_base.fixtures.ts} (87%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/basic_document_migration.test.ts (98%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/conversion_failures.test.ts (98%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/create_index.test.ts (99%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/document_cleanup.test.ts (98%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/jest.integration.config.js (96%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/mapping_version_conflict.test.ts (96%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/rerun_same_version.test.ts (98%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/standard_workflow.test.ts (99%) rename src/core/server/integration_tests/saved_objects/migrations/{zero_downtime => zdt_1}/update_mappings.test.ts (96%) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/jest.integration.config.js create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts similarity index 87% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts rename to src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts index 3cf5e499eaa6e..dde1bba362fce 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { SavedObjectsModelVersion, SavedObjectMigrationFn } from '@kbn/core-saved-objects-server'; import { createType } from '../test_utils'; import { type KibanaMigratorTestKitParams } from '../kibana_migrator_test_kit'; @@ -29,6 +29,8 @@ export const dummyModelVersion: SavedObjectsModelVersion = { }, }; +export const dummyMigration: SavedObjectMigrationFn = (doc) => doc; + export const getFooType = () => { return createType({ name: 'foo', @@ -130,3 +132,18 @@ export const getExcludedType = () => { }, }); }; + +export const getLegacyType = () => { + return createType({ + name: 'legacy', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + migrations: { + '7.0.0': dummyMigration, + '7.5.0': dummyMigration, + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/basic_document_migration.test.ts similarity index 98% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/basic_document_migration.test.ts index 83f7fda262854..9122cb1bd6c10 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/basic_document_migration.test.ts @@ -14,7 +14,11 @@ import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server import '../jest_matchers'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; +import { + getBaseMigratorParams, + getSampleAType, + getSampleBType, +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/conversion_failures.test.ts similarity index 98% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/conversion_failures.test.ts index 4f1b0a4bfe468..750753d52c66b 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/conversion_failures.test.ts @@ -14,7 +14,11 @@ import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server import '../jest_matchers'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; +import { + getBaseMigratorParams, + getSampleAType, + getSampleBType, +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'conversion_failures.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/create_index.test.ts similarity index 99% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/create_index.test.ts index bee9013544b72..18b9aecc39cad 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/create_index.test.ts @@ -12,7 +12,7 @@ import '../jest_matchers'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; +import { getBaseMigratorParams, getFooType, getBarType } from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'create_index.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/document_cleanup.test.ts similarity index 98% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/document_cleanup.test.ts index b321a400684b7..98e85b3029f18 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/document_cleanup.test.ts @@ -21,7 +21,7 @@ import { getFooType, getBarType, dummyModelVersion, -} from './base.fixtures'; +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'document_cleanup.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/jest.integration.config.js similarity index 96% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/jest.integration.config.js index 4772e43faa148..04b7ddfc4377a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/jest.integration.config.js @@ -13,7 +13,7 @@ module.exports = { // see https://github.com/elastic/kibana/pull/130255/ preset: '@kbn/test/jest_integration', rootDir: '../../../../../../..', - roots: ['/src/core/server/integration_tests/saved_objects/migrations/zero_downtime'], + roots: ['/src/core/server/integration_tests/saved_objects/migrations/zdt_1'], // must override to match all test given there is no `integration_tests` subfolder testMatch: ['**/*.test.{js,mjs,ts,tsx}'], }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/mapping_version_conflict.test.ts similarity index 96% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/mapping_version_conflict.test.ts index 55ff37d47b080..50edd85babe8e 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/mapping_version_conflict.test.ts @@ -12,7 +12,12 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h import '../jest_matchers'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; +import { + getBaseMigratorParams, + getFooType, + getBarType, + dummyModelVersion, +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/rerun_same_version.test.ts similarity index 98% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/rerun_same_version.test.ts index 2be7c93965693..42ac41a84c92c 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/rerun_same_version.test.ts @@ -12,7 +12,7 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h import '../jest_matchers'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; +import { getBaseMigratorParams, getFooType, getBarType } from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'rerun_same_version.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/standard_workflow.test.ts similarity index 99% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/standard_workflow.test.ts index 2fbbe7e090267..2e08df454f111 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/standard_workflow.test.ts @@ -19,7 +19,7 @@ import { getSampleAType, getSampleBType, dummyModelVersion, -} from './base.fixtures'; +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'standard_workflow.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/update_mappings.test.ts similarity index 96% rename from src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts rename to src/core/server/integration_tests/saved_objects/migrations/zdt_1/update_mappings.test.ts index 8117b3b68137f..f8f49147ef753 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_1/update_mappings.test.ts @@ -12,7 +12,12 @@ import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-h import '../jest_matchers'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; -import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; +import { + getBaseMigratorParams, + getFooType, + getBarType, + dummyModelVersion, +} from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts new file mode 100644 index 0000000000000..ee59b2290bc08 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { + getBaseMigratorParams, + getSampleAType, + getLegacyType, +} from '../fixtures/zdt_base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); + +describe('ZDT with v2 compat - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getLegacyType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const legacyTypeObjs = range(5).map((number) => ({ + id: `legacy-${number}`, + type: 'legacy', + attributes: { + someField: `legacy ${number}`, + }, + })); + + await savedObjectsRepository.bulkCreate(legacyTypeObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const legacyType = getLegacyType(); + + // typeA -> we add a new field and bump the model version by one with a migration + + typeA.mappings.properties = { + ...typeA.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + someAddedField: `${doc.attributes.keyword}-mig`, + }, + }, + }; + }, + down: jest.fn(), + }, + addedMappings: { + someAddedField: { type: 'keyword' }, + }, + }, + }, + }; + + // legacyType -> we add a new migration + + legacyType.mappings.properties = { + ...legacyType.mappings.properties, + newField: { type: 'text' }, + }; + + legacyType.migrations = { + ...legacyType.migrations, + '8.0.0': (document) => { + return { + ...document, + attributes: { + ...document.attributes, + newField: `populated ${document.id}`, + }, + }; + }, + }; + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, legacyType], + }); + + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + sample_a: typeA.mappings, + legacy: legacyType.mappings, + }) + ); + + expect(mappingMeta.docVersions).toEqual({ + sample_a: '10.2.0', + legacy: '8.0.0', + }); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' }); + const { saved_objects: legacyDocs } = await savedObjectsRepository.find({ type: 'legacy' }); + + expect(sampleADocs).toHaveLength(5); + expect(legacyDocs).toHaveLength(5); + + const sampleAData = sortBy(sampleADocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleAData).toEqual([ + { + id: 'a-0', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_0', someAddedField: 'a_0-mig' }, + }, + { + id: 'a-1', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_1', someAddedField: 'a_1-mig' }, + }, + { + id: 'a-2', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_2', someAddedField: 'a_2-mig' }, + }, + { + id: 'a-3', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_3', someAddedField: 'a_3-mig' }, + }, + { + id: 'a-4', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_4', someAddedField: 'a_4-mig' }, + }, + ]); + + const sampleBData = sortBy(legacyDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleBData).toEqual([ + { + id: 'legacy-0', + type: 'legacy', + attributes: { someField: `legacy 0`, newField: `populated legacy-0` }, + }, + { + id: 'legacy-1', + type: 'legacy', + attributes: { someField: `legacy 1`, newField: `populated legacy-1` }, + }, + { + id: 'legacy-2', + type: 'legacy', + attributes: { someField: `legacy 2`, newField: `populated legacy-2` }, + }, + { + id: 'legacy-3', + type: 'legacy', + attributes: { someField: `legacy 3`, newField: `populated legacy-3` }, + }, + { + id: 'legacy-4', + type: 'legacy', + attributes: { someField: `legacy 4`, newField: `populated legacy-4` }, + }, + ]); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('Starting to process 10 documents'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts new file mode 100644 index 0000000000000..073d193145aa3 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import '../jest_matchers'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getLegacyType } from '../fixtures/zdt_base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'create_index.test.log'); + +describe('ZDT with v2 compat - running on a fresh cluster', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + it('create the index with the correct mappings and meta', async () => { + const fooType = getFooType(); + const legacyType = getLegacyType(); + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, legacyType], + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + legacy: legacyType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + docVersions: { + foo: '10.2.0', + legacy: '7.5.0', + }, + mappingVersions: { + foo: '10.2.0', + legacy: '7.5.0', + }, + migrationState: expect.objectContaining({ + convertingDocuments: false, + }), + }); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> CREATE_TARGET_INDEX'); + expect(records).toContainLogEntry('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expect(records).toContainLogEntry('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DONE'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/jest.integration.config.js b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/jest.integration.config.js new file mode 100644 index 0000000000000..33c212f19a68e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/jest.integration.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + // TODO replace the line below with + // preset: '@kbn/test/jest_integration_node + // to do so, we must fix all integration tests first + // see https://github.com/elastic/kibana/pull/130255/ + preset: '@kbn/test/jest_integration', + rootDir: '../../../../../../..', + roots: ['/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat'], + // must override to match all test given there is no `integration_tests` subfolder + testMatch: ['**/*.test.{js,mjs,ts,tsx}'], +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts new file mode 100644 index 0000000000000..03994097c2c58 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { + getBaseMigratorParams, + getFooType, + getLegacyType, + dummyModelVersion, + dummyMigration, +} from '../fixtures/zdt_base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); + +describe('ZDT with v2 compat - basic mapping update', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const legacyType = getLegacyType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [fooType, legacyType], + }); + await runMigrations(); + }; + + it('updates the mappings and the meta', async () => { + await createBaseline(); + + const fooType = getFooType(); + const legacyType = getLegacyType(); + + // increasing the model version of the types + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + fooType.mappings.properties = { + ...fooType.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + legacyType.migrations = { + ...legacyType.migrations, + '8.0.0': dummyMigration, + }; + legacyType.mappings.properties = { + ...legacyType.mappings.properties, + anotherAddedField: { type: 'boolean' }, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, legacyType], + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + legacy: legacyType.mappings, + }) + ); + + expect(mappingMeta.mappingVersions).toEqual({ + foo: '10.3.0', + legacy: '8.0.0', + }); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); From 8a2e3e147fd2fdf2373932b3127c4e3fa5c2606b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 1 May 2023 12:23:21 +0200 Subject: [PATCH 6/7] add compat integration tests --- .../migrations/fixtures/zdt_base.fixtures.ts | 6 +- .../basic_document_migration.test.ts | 2 +- .../zdt_v2_compat/create_index.test.ts | 2 +- .../switch_to_model_version.test.ts | 182 ++++++++++++++++++ .../zdt_v2_compat/update_mappings.test.ts | 4 +- 5 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/switch_to_model_version.test.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts index dde1bba362fce..53c7c1d11613d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/fixtures/zdt_base.fixtures.ts @@ -29,7 +29,7 @@ export const dummyModelVersion: SavedObjectsModelVersion = { }, }; -export const dummyMigration: SavedObjectMigrationFn = (doc) => doc; +export const noopMigration: SavedObjectMigrationFn = (doc) => doc; export const getFooType = () => { return createType({ @@ -142,8 +142,8 @@ export const getLegacyType = () => { }, }, migrations: { - '7.0.0': dummyMigration, - '7.5.0': dummyMigration, + '7.0.0': noopMigration, + '7.5.0': noopMigration, }, }); }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts index ee59b2290bc08..14d707e51a7ec 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/basic_document_migration.test.ts @@ -20,7 +20,7 @@ import { getLegacyType, } from '../fixtures/zdt_base.fixtures'; -export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); +const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); describe('ZDT with v2 compat - basic document migration', () => { let esServer: TestElasticsearchUtils['es']; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts index 073d193145aa3..4933c5b712c21 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/create_index.test.ts @@ -14,7 +14,7 @@ import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; import { getBaseMigratorParams, getFooType, getLegacyType } from '../fixtures/zdt_base.fixtures'; -export const logFilePath = Path.join(__dirname, 'create_index.test.log'); +const logFilePath = Path.join(__dirname, 'create_index.test.log'); describe('ZDT with v2 compat - running on a fresh cluster', () => { let esServer: TestElasticsearchUtils['es']; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/switch_to_model_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/switch_to_model_version.test.ts new file mode 100644 index 0000000000000..60aa99da52e72 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/switch_to_model_version.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile, createType } from '../test_utils'; +import { getBaseMigratorParams, noopMigration } from '../fixtures/zdt_base.fixtures'; + +const logFilePath = Path.join(__dirname, 'switch_to_model_version.test.log'); + +describe('ZDT with v2 compat - type switching from migration to model version', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const typeBefore = createType({ + name: 'switching_type', + mappings: { + properties: { + text: { type: 'text' }, + keyword: { type: 'keyword' }, + }, + }, + migrations: { + '7.0.0': noopMigration, + '7.5.0': noopMigration, + }, + }); + + const typeAfter = createType({ + name: 'switching_type', + mappings: { + properties: { + text: { type: 'text' }, + keyword: { type: 'keyword' }, + newField1: { type: 'text' }, + newField2: { type: 'text' }, + }, + }, + migrations: { + '7.0.0': noopMigration, + '7.5.0': noopMigration, + '7.9.0': (doc) => { + return { + ...doc, + attributes: { + ...doc.attributes, + newField1: `new1 ${doc.id}`, + }, + }; + }, + }, + switchToModelVersionAt: '8.0.0', + modelVersions: { + 1: { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + newField2: `new2 ${doc.id}`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + }, + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [typeBefore], + }); + await runMigrations(); + + const sampleObjs = range(5).map((number) => ({ + id: `doc-${number}`, + type: 'switching_type', + attributes: { + text: `text ${number}`, + keyword: `kw ${number}`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeAfter], + }); + + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + switching_type: typeAfter.mappings, + }) + ); + + expect(mappingMeta.docVersions).toEqual({ + switching_type: '10.1.0', + }); + + const { saved_objects: sampleDocs } = await savedObjectsRepository.find({ + type: 'switching_type', + }); + + expect(sampleDocs).toHaveLength(5); + + const sampleData = sortBy(sampleDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleData).toEqual( + range(5).map((i) => ({ + id: `doc-${i}`, + type: 'switching_type', + attributes: { + text: `text ${i}`, + keyword: `kw ${i}`, + newField1: `new1 doc-${i}`, + newField2: `new2 doc-${i}`, + }, + })) + ); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts index 03994097c2c58..ea25e6549c189 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_v2_compat/update_mappings.test.ts @@ -17,7 +17,7 @@ import { getFooType, getLegacyType, dummyModelVersion, - dummyMigration, + noopMigration, } from '../fixtures/zdt_base.fixtures'; export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); @@ -75,7 +75,7 @@ describe('ZDT with v2 compat - basic mapping update', () => { legacyType.migrations = { ...legacyType.migrations, - '8.0.0': dummyMigration, + '8.0.0': noopMigration, }; legacyType.mappings.properties = { ...legacyType.mappings.properties, From 36cb6c63a877f3738ad06df7812ba78c9d815e73 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 2 May 2023 12:31:13 +0200 Subject: [PATCH 7/7] review NITs --- .../src/model_version/version_map.ts | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts index e4d2a86077065..aafac41df794a 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -10,8 +10,26 @@ import Semver from 'semver'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import { assertValidModelVersion, modelVersionToVirtualVersion } from './conversion'; -export type ModelVersionMap = Record; +/** + * Represents the virtual version of a given SO type. + * The virtual version is a compatibility format between the old + * migration system's versioning, based on the stack version, and the new model versioning. + * + * A virtual version is a plain semver version. Depending on its major version value, the + * underlying version can be the following: + * - Major < 10: Old migrations system (stack versions), using the equivalent value (e.g `8.7.0` => migration version `8.7.0`) + * - Major == 10: Model versions, using the `10.{modelVersion}.0` format (e.g `10.3.0` => model version 3) + */ export type VirtualVersion = string; + +/** + * A map of SO type name to Model Version. + */ +export type ModelVersionMap = Record; + +/** + * A map of SO type name to {@link VirtualVersion}. + */ export type VirtualVersionMap = Record; /** @@ -45,7 +63,7 @@ export const getModelVersionMapForTypes = (types: SavedObjectsType[]): ModelVers /** * Returns the current virtual version for the given type. - * If will either be the latest model version if the type + * It will either be the latest model version if the type * already switched to using them (switchToModelVersionAt is set), * or the latest migration version for the type otherwise. */