[dot-kibana-split] Allow relocating SO to different indices during mi…

…gration (#154846) Clone of #154151 Baseline for the `dot-kibana-split` feature branch
elastic · Apr 13, 2023 · 0c154cb · 0c154cb
1 parent d83f1fe
commit 0c154cb
Show file tree

Hide file tree

Showing 44 changed files with 5,958 additions and 429 deletions.
diff --git a/...ts/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts b/...ts/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts
@@ -22,8 +22,9 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m
 import { kibanaMigratorMock } from '../mocks';
 import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal';
 import {
-  ISavedObjectsEncryptionExtension,
-  SavedObjectsRawDocSource,
+  MAIN_SAVED_OBJECT_INDEX,
+  type ISavedObjectsEncryptionExtension,
+  type SavedObjectsRawDocSource,
 } from '@kbn/core-saved-objects-server';
 import {
   bulkCreateSuccess,
@@ -41,8 +42,8 @@ import {
   mockVersion,
   mockVersionProps,
   MULTI_NAMESPACE_ENCRYPTED_TYPE,
-  TypeIdTuple,
   updateSuccess,
+  type TypeIdTuple,
 } from '../test_helpers/repository.test.common';
 import { savedObjectsExtensionsMock } from '../mocks/saved_objects_extensions.mock';
 
@@ -633,7 +634,7 @@ describe('SavedObjectsRepository Encryption Extension', () => {
           total: 2,
           hits: [
             {
-              _index: '.kibana',
+              _index: MAIN_SAVED_OBJECT_INDEX,
               _id: `${space ? `${space}:` : ''}${encryptedSO.type}:${encryptedSO.id}`,
               _score: 1,
               ...mockVersionProps,
@@ -643,7 +644,7 @@ describe('SavedObjectsRepository Encryption Extension', () => {
               },
             },
             {
-              _index: '.kibana',
+              _index: MAIN_SAVED_OBJECT_INDEX,
               _id: `${space ? `${space}:` : ''}index-pattern:logstash-*`,
               _score: 2,
               ...mockVersionProps,

diff --git a/...ages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts b/...ages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts
@@ -47,13 +47,14 @@ import type {
   SavedObjectsBulkDeleteObject,
   SavedObjectsBulkDeleteOptions,
 } from '@kbn/core-saved-objects-api-server';
-import type {
-  SavedObjectsRawDoc,
-  SavedObjectsRawDocSource,
-  SavedObjectUnsanitizedDoc,
-  SavedObject,
-  SavedObjectReference,
-  BulkResolveError,
+import {
+  type SavedObjectsRawDoc,
+  type SavedObjectsRawDocSource,
+  type SavedObjectUnsanitizedDoc,
+  type SavedObject,
+  type SavedObjectReference,
+  type BulkResolveError,
+  MAIN_SAVED_OBJECT_INDEX,
 } from '@kbn/core-saved-objects-server';
 import { ALL_NAMESPACES_STRING } from '@kbn/core-saved-objects-utils-server';
 import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
@@ -4115,7 +4116,7 @@ describe('SavedObjectsRepository', () => {
           body: {
             _id: params.id,
             ...mockVersionProps,
-            _index: '.kibana',
+            _index: MAIN_SAVED_OBJECT_INDEX,
             get: {
               found: true,
               _source: {
@@ -4419,7 +4420,7 @@ describe('SavedObjectsRepository', () => {
             body: {
               _id: params.id,
               ...mockVersionProps,
-              _index: '.kibana',
+              _index: MAIN_SAVED_OBJECT_INDEX,
               get: {
                 found: true,
                 _source: {

diff --git a/...objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts b/...objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts
@@ -9,19 +9,20 @@
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { schema } from '@kbn/config-schema';
 import { loggerMock } from '@kbn/logging-mocks';
-import { Payload } from 'elastic-apm-node';
-import type {
-  AuthorizationTypeEntry,
-  AuthorizeAndRedactMultiNamespaceReferencesParams,
-  CheckAuthorizationResult,
-  ISavedObjectsSecurityExtension,
-  SavedObjectsMappingProperties,
-  SavedObjectsRawDocSource,
-  SavedObjectsType,
-  SavedObjectsTypeMappingDefinition,
-  SavedObject,
-  SavedObjectReference,
-  AuthorizeFindParams,
+import type { Payload } from 'elastic-apm-node';
+import {
+  type AuthorizationTypeEntry,
+  type AuthorizeAndRedactMultiNamespaceReferencesParams,
+  type CheckAuthorizationResult,
+  type ISavedObjectsSecurityExtension,
+  type SavedObjectsMappingProperties,
+  type SavedObjectsRawDocSource,
+  type SavedObjectsType,
+  type SavedObjectsTypeMappingDefinition,
+  type SavedObject,
+  type SavedObjectReference,
+  type AuthorizeFindParams,
+  MAIN_SAVED_OBJECT_INDEX,
 } from '@kbn/core-saved-objects-server';
 import type {
   SavedObjectsBaseOptions,
@@ -47,9 +48,9 @@ import {
 } from '@kbn/core-elasticsearch-client-server-mocks';
 import { DocumentMigrator } from '@kbn/core-saved-objects-migration-server-internal';
 import {
-  AuthorizeAndRedactInternalBulkResolveParams,
-  GetFindRedactTypeMapParams,
-  AuthorizationTypeMap,
+  type AuthorizeAndRedactInternalBulkResolveParams,
+  type GetFindRedactTypeMapParams,
+  type AuthorizationTypeMap,
   SavedObjectsErrorHelpers,
 } from '@kbn/core-saved-objects-server';
 import { mockGetSearchDsl } from '../lib/repository.test.mock';
@@ -592,8 +593,6 @@ export const getMockBulkCreateResponse = (
     items: objects.map(
       ({ type, id, originId, attributes, references, migrationVersion, typeMigrationVersion }) => ({
         create: {
-          // status: 1,
-          // _index: '.kibana',
           _id: `${namespace ? `${namespace}:` : ''}${type}:${id}`,
           _source: {
             [type]: attributes,
@@ -714,7 +713,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
       total: 4,
       hits: [
         {
-          _index: '.kibana',
+          _index: MAIN_SAVED_OBJECT_INDEX,
           _id: `${namespace ? `${namespace}:` : ''}index-pattern:logstash-*`,
           _score: 1,
           ...mockVersionProps,
@@ -731,7 +730,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
           },
         },
         {
-          _index: '.kibana',
+          _index: MAIN_SAVED_OBJECT_INDEX,
           _id: `${namespace ? `${namespace}:` : ''}config:6.0.0-alpha1`,
           _score: 2,
           ...mockVersionProps,
@@ -746,7 +745,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
           },
         },
         {
-          _index: '.kibana',
+          _index: MAIN_SAVED_OBJECT_INDEX,
           _id: `${namespace ? `${namespace}:` : ''}index-pattern:stocks-*`,
           _score: 3,
           ...mockVersionProps,
@@ -762,7 +761,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
           },
         },
         {
-          _index: '.kibana',
+          _index: MAIN_SAVED_OBJECT_INDEX,
           _id: `${NAMESPACE_AGNOSTIC_TYPE}:something`,
           _score: 4,
           ...mockVersionProps,

diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts
@@ -14,6 +14,7 @@ export {
   getTypes,
   type IndexMapping,
   type IndexMappingMeta,
+  type IndexTypesMap,
   type SavedObjectsTypeMappingDefinitions,
   type IndexMappingMigrationStateMeta,
 } from './src/mappings';

diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts
@@ -11,5 +11,6 @@ export type {
   SavedObjectsTypeMappingDefinitions,
   IndexMappingMeta,
   IndexMapping,
+  IndexTypesMap,
   IndexMappingMigrationStateMeta,
 } from './types';
diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts
@@ -55,6 +55,9 @@ export interface IndexMapping {
   _meta?: IndexMappingMeta;
 }
 
+/** @internal */
+export type IndexTypesMap = Record<string, string[]>;
+
 /** @internal */
 export interface IndexMappingMeta {
   /**
@@ -65,6 +68,12 @@ export interface IndexMappingMeta {
    * @remark: Only defined for indices using the v2 migration algorithm.
    */
   migrationMappingPropertyHashes?: { [k: string]: string };
+  /**
+   * A map that tells what are the SO types stored in each index
+   *
+   * @remark: Only defined for indices using the v2 migration algorithm.
+   */
+  indexTypesMap?: IndexTypesMap;
   /**
    * The current model versions of the mapping of the index.
    *

diff --git a/...re/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts b/...re/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts
@@ -69,7 +69,11 @@ export type MigrationStatus =
 /** @internal */
 export type MigrationResult =
   | { status: 'skipped' }
-  | { status: 'patched' }
+  | {
+      status: 'patched';
+      destIndex: string;
+      elapsedMs: number;
+    }
   | {
       status: 'migrated';
       destIndex: string;

diff --git a/...aved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts b/...aved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts
@@ -6,7 +6,10 @@
  * Side Public License, v 1.
  */
 
-import { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
+import {
+  type ISavedObjectTypeRegistry,
+  MAIN_SAVED_OBJECT_INDEX,
+} from '@kbn/core-saved-objects-server';
 import { getIndexForType } from './get_index_for_type';
 
 const createTypeRegistry = () => {
@@ -17,7 +20,7 @@ const createTypeRegistry = () => {
 
 describe('getIndexForType', () => {
   const kibanaVersion = '8.0.0';
-  const defaultIndex = '.kibana';
+  const defaultIndex = MAIN_SAVED_OBJECT_INDEX;
   let typeRegistry: ReturnType<typeof createTypeRegistry>;
 
   beforeEach(() => {