[dot-kibana-split] Split analyst experience SOs into `.kibana_analyti…

…cs` (#155140) ## Summary **This is targeting a feature branch** Part of #154888 This PR aims at moving the following saved object types to a new `.kibana_analytics` system index. - canvas-element - canvas-workpad-template - canvas-workpad - dashboard - graph-workspace - index-pattern - kql-telemetry - lens - lens-ui-telemetry - map - search - search-session - search-telemetry - url - visualization --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: pgayvallet <pierre.gayvallet@elastic.co>
elastic · Apr 19, 2023 · b416f1a · b416f1a
1 parent cf8a056
commit b416f1a
Show file tree

Hide file tree

Showing 50 changed files with 1,847 additions and 269 deletions.
diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts
@@ -58,6 +58,7 @@ export {
   INGEST_SAVED_OBJECT_INDEX,
   ALERTING_CASES_SAVED_OBJECT_INDEX,
   SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
+  ANALYTICS_SAVED_OBJECT_INDEX,
   SavedObjectsIndexPatterns,
 } from './src/saved_objects_index_pattern';
 export type {

diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts
@@ -18,10 +18,12 @@ export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_
 export const INGEST_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_ingest`;
 export const ALERTING_CASES_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_alerting_cases`;
 export const SECURITY_SOLUTION_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_security_solution`;
+export const ANALYTICS_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_analytics`;
 export const SavedObjectsIndexPatterns = [
   MAIN_SAVED_OBJECT_INDEX,
   TASK_MANAGER_SAVED_OBJECT_INDEX,
   ALERTING_CASES_SAVED_OBJECT_INDEX,
   INGEST_SAVED_OBJECT_INDEX,
   SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
+  ANALYTICS_SAVED_OBJECT_INDEX,
 ];
diff --git a/...erver/integration_tests/saved_objects/migrations/group1/7_13_0_transform_failures.test.ts b/...erver/integration_tests/saved_objects/migrations/group1/7_13_0_transform_failures.test.ts
@@ -24,6 +24,7 @@ import {
 import {
   MAIN_SAVED_OBJECT_INDEX,
   TASK_MANAGER_SAVED_OBJECT_INDEX,
+  ANALYTICS_SAVED_OBJECT_INDEX,
 } from '@kbn/core-saved-objects-server';
 
 const migrationDocLink = getMigrationDocLink().resolveMigrationFailures;
@@ -118,7 +119,11 @@ describe('migration v2', () => {
 
       const esClient: ElasticsearchClient = esServer.es.getClient();
       const docs = await esClient.search({
-        index: [MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX],
+        index: [
+          MAIN_SAVED_OBJECT_INDEX,
+          TASK_MANAGER_SAVED_OBJECT_INDEX,
+          ANALYTICS_SAVED_OBJECT_INDEX,
+        ],
         _source: false,
         fields: ['_id'],
         size: 50,

diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts
@@ -131,7 +131,7 @@ describe('migration v2', () => {
         expect(
           records.find((rec) =>
             rec.message.startsWith(
-              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715286 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
+              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715296 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
             )
           )
         ).toBeDefined();

diff --git a/src/plugins/content_management/docs/content_onboarding.md b/src/plugins/content_management/docs/content_onboarding.md
@@ -239,7 +239,7 @@ import { cmServicesDefinition } from '../../common/content_management/cm_service
  * that we won't leak any additional fields in our Response, even when the SO client adds new fields to its responses.
  */
 function savedObjectToMapItem(
-  savedObject: SavedObject<MapSavedObjectAttributes>
+  savedObject: SavedObject<MapAttributes>
 ): MapItem {
   const {
     id,
@@ -293,7 +293,7 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
       alias_purpose: aliasPurpose,
       alias_target_id: aliasTargetId,
       outcome,
-    } = await soClient.resolve<MapSavedObjectAttributes>(SO_TYPE, id);
+    } = await soClient.resolve<MapAttributes>(SO_TYPE, id);
 
     const response: MapGetOut = {
       item: savedObjectToMapItem(savedObject),
@@ -327,8 +327,8 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
 
     // Validate input (data & options) & UP transform them to the latest version
     const { value: dataToLatest, error: dataError } = transforms.create.in.data.up<
-      MapSavedObjectAttributes,
-      MapSavedObjectAttributes
+      MapAttributes,
+      MapAttributes
     >(data);
     if (dataError) {
       throw Boom.badRequest(`Invalid payload. ${dataError.message}`);
@@ -345,7 +345,7 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
     //   - both are on the latest version
 
     // Save data in DB
-    const savedObject = await soClient.create<MapSavedObjectAttributes>(
+    const savedObject = await soClient.create<MapAttributes>(
       SO_TYPE,
       dataToLatest,
       optionsToLatest

diff --git a/src/plugins/dashboard/server/dashboard_saved_object/dashboard_saved_object.ts b/src/plugins/dashboard/server/dashboard_saved_object/dashboard_saved_object.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import {
   createDashboardSavedObjectTypeMigrations,
@@ -18,6 +19,7 @@ export const createDashboardSavedObjectType = ({
   migrationDeps: DashboardSavedObjectTypeMigrationsDeps;
 }): SavedObjectsType => ({
   name: 'dashboard',
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   hidden: false,
   namespaceType: 'multiple-isolated',
   convertToMultiNamespaceTypeVersion: '8.0.0',

diff --git a/src/plugins/dashboard/tsconfig.json b/src/plugins/dashboard/tsconfig.json
@@ -56,6 +56,7 @@
     "@kbn/saved-objects-finder-plugin",
     "@kbn/saved-objects-management-plugin",
     "@kbn/shared-ux-button-toolbar",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/plugins/data/server/saved_objects/kql_telemetry.ts b/src/plugins/data/server/saved_objects/kql_telemetry.ts
@@ -6,11 +6,13 @@
  * Side Public License, v 1.
  */
 
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { SCHEMA_KQL_TELEMETRY_V8_8_0 } from './schemas/kql_telemetry';
 
 export const kqlTelemetry: SavedObjectsType = {
   name: 'kql-telemetry',
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   namespaceType: 'agnostic',
   hidden: false,
   mappings: {

diff --git a/src/plugins/data/server/saved_objects/query.ts b/src/plugins/data/server/saved_objects/query.ts
@@ -6,12 +6,14 @@
  * Side Public License, v 1.
  */
 
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { savedQueryMigrations } from './migrations/query';
 import { SCHEMA_QUERY_V8_8_0 } from './schemas/query';
 
 export const querySavedObjectType: SavedObjectsType = {
   name: 'query',
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   hidden: false,
   namespaceType: 'multiple-isolated',
   convertToMultiNamespaceTypeVersion: '8.0.0',

diff --git a/src/plugins/data/server/saved_objects/search_telemetry.ts b/src/plugins/data/server/saved_objects/search_telemetry.ts
@@ -6,12 +6,14 @@
  * Side Public License, v 1.
  */
 
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { migrate712 } from './migrations/to_v7_12_0';
 import { SCHEMA_SEARCH_TELEMETRY_V8_8_0 } from './schemas/search_telemetry';
 
 export const searchTelemetry: SavedObjectsType = {
   name: 'search-telemetry',
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   namespaceType: 'agnostic',
   hidden: false,
   mappings: {

diff --git a/src/plugins/data/server/search/saved_objects/search_session.ts b/src/plugins/data/server/search/saved_objects/search_session.ts
@@ -7,12 +7,14 @@
  */
 
 import { schema } from '@kbn/config-schema';
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { SEARCH_SESSION_TYPE } from '../../../common';
 import { searchSessionSavedObjectMigrations } from './search_session_migration';
 
 export const searchSessionSavedObjectType: SavedObjectsType = {
   name: SEARCH_SESSION_TYPE,
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   namespaceType: 'single',
   hidden: true,
   mappings: {

diff --git a/src/plugins/data/tsconfig.json b/src/plugins/data/tsconfig.json
@@ -47,6 +47,7 @@
     "@kbn/config",
     "@kbn/config-schema",
     "@kbn/core-application-browser",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/plugins/data_views/server/saved_objects/data_views.ts b/src/plugins/data_views/server/saved_objects/data_views.ts
@@ -7,11 +7,13 @@
  */
 
 import type { SavedObjectsType } from '@kbn/core/server';
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { indexPatternSavedObjectTypeMigrations } from './index_pattern_migrations';
 import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../common';
 
 export const dataViewSavedObjectType: SavedObjectsType = {
   name: DATA_VIEW_SAVED_OBJECT_TYPE,
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   hidden: false,
   namespaceType: 'multiple',
   convertToMultiNamespaceTypeVersion: '8.0.0',

diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json
@@ -28,6 +28,7 @@
     "@kbn/utility-types-jest",
     "@kbn/safer-lodash-set",
     "@kbn/core-http-server",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/plugins/saved_search/server/saved_objects/search.ts b/src/plugins/saved_search/server/saved_objects/search.ts
@@ -7,6 +7,7 @@
  */
 
 import { schema } from '@kbn/config-schema';
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common';
 import { VIEW_MODE } from '../../common';
@@ -17,6 +18,7 @@ export function getSavedSearchObjectType(
 ): SavedObjectsType {
   return {
     name: 'search',
+    indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
     hidden: false,
     namespaceType: 'multiple-isolated',
     convertToMultiNamespaceTypeVersion: '8.0.0',

diff --git a/src/plugins/saved_search/tsconfig.json b/src/plugins/saved_search/tsconfig.json
@@ -17,6 +17,7 @@
     "@kbn/saved-objects-tagging-oss-plugin",
     "@kbn/i18n",
     "@kbn/config-schema",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/plugins/share/server/url_service/saved_objects/register_url_service_saved_object_type.ts b/src/plugins/share/server/url_service/saved_objects/register_url_service_saved_object_type.ts
@@ -11,6 +11,7 @@ import type {
   SavedObjectsServiceSetup,
   SavedObjectsType,
 } from '@kbn/core/server';
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import type { LocatorData } from '../../../common/url_service';
 import type { ServerUrlService } from '..';
 
@@ -20,6 +21,7 @@ export const registerUrlServiceSavedObjectType = (
 ) => {
   const urlSavedObjectType: SavedObjectsType = {
     name: 'url',
+    indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
     namespaceType: 'single',
     hidden: false,
     management: {

diff --git a/src/plugins/share/tsconfig.json b/src/plugins/share/tsconfig.json
@@ -13,6 +13,7 @@
     "@kbn/i18n-react",
     "@kbn/config-schema",
     "@kbn/core-custom-branding-browser",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/src/plugins/visualizations/server/saved_objects/visualization.ts b/src/plugins/visualizations/server/saved_objects/visualization.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SavedObjectsType } from '@kbn/core/server';
 import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common';
 import { getAllMigrations } from '../migrations/visualization_saved_object_migrations';
@@ -14,6 +15,7 @@ export const getVisualizationSavedObjectType = (
   getSearchSourceMigrations: () => MigrateFunctionsObject
 ): SavedObjectsType => ({
   name: 'visualization',
+  indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
   hidden: false,
   namespaceType: 'multiple-isolated',
   convertToMultiNamespaceTypeVersion: '8.0.0',

diff --git a/src/plugins/visualizations/tsconfig.json b/src/plugins/visualizations/tsconfig.json
@@ -54,6 +54,7 @@
     "@kbn/shared-ux-router",
     "@kbn/saved-objects-management-plugin",
     "@kbn/saved-objects-finder-plugin",
+    "@kbn/core-saved-objects-server",
   ],
   "exclude": [
     "target/**/*",

diff --git a/test/api_integration/apis/kql_telemetry/kql_telemetry.ts b/test/api_integration/apis/kql_telemetry/kql_telemetry.ts
@@ -8,6 +8,7 @@
 
 import expect from '@kbn/expect';
 import { get } from 'lodash';
+import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
@@ -27,7 +28,7 @@ export default function ({ getService }: FtrProviderContext) {
       );
     });
 
-    it('should increment the opt *in* counter in the .kibana/kql-telemetry document', async () => {
+    it('should increment the opt *in* counter in the .kibana_analytics/kql-telemetry document', async () => {
       await supertest
         .post('/api/kibana/kql_opt_in_stats')
         .set('content-type', 'application/json')
@@ -36,7 +37,7 @@ export default function ({ getService }: FtrProviderContext) {
 
       return es
         .search({
-          index: '.kibana',
+          index: ANALYTICS_SAVED_OBJECT_INDEX,
           q: 'type:kql-telemetry',
         })
         .then((response) => {
@@ -45,7 +46,7 @@ export default function ({ getService }: FtrProviderContext) {
         });
     });
 
-    it('should increment the opt *out* counter in the .kibana/kql-telemetry document', async () => {
+    it('should increment the opt *out* counter in the .kibana_analytics/kql-telemetry document', async () => {
       await supertest
         .post('/api/kibana/kql_opt_in_stats')
         .set('content-type', 'application/json')
@@ -54,7 +55,7 @@ export default function ({ getService }: FtrProviderContext) {
 
       return es
         .search({
-          index: '.kibana',
+          index: ANALYTICS_SAVED_OBJECT_INDEX,
           q: 'type:kql-telemetry',
         })
         .then((response) => {

diff --git a/test/api_integration/apis/saved_objects/delete_unknown_types.ts b/test/api_integration/apis/saved_objects/delete_unknown_types.ts
@@ -7,6 +7,10 @@
  */
 
 import expect from '@kbn/expect';
+import {
+  MAIN_SAVED_OBJECT_INDEX,
+  ANALYTICS_SAVED_OBJECT_INDEX,
+} from '@kbn/core-saved-objects-server';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -32,7 +36,7 @@ export default function ({ getService }: FtrProviderContext) {
 
     const fetchIndexContent = async () => {
       const body = await es.search<{ type: string }>({
-        index: '.kibana',
+        index: [MAIN_SAVED_OBJECT_INDEX, ANALYTICS_SAVED_OBJECT_INDEX],
         body: {
           size: 100,
         },

diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/data.json
@@ -2,7 +2,7 @@
   "type": "doc",
   "value": {
     "id": "index-pattern:8963ca30-3224-11e8-a572-ffca06da1357",
-    "index": ".kibana",
+    "index": ".kibana_analytics",
     "source": {
       "coreMigrationVersion": "7.14.0",
       "index-pattern": {
@@ -25,7 +25,7 @@
   "type": "doc",
   "value": {
     "id": "search:960372e0-3224-11e8-a572-ffca06da1357",
-    "index": ".kibana",
+    "index": ".kibana_analytics",
     "source": {
       "coreMigrationVersion": "7.14.0",
       "migrationVersion": {
@@ -67,7 +67,7 @@
   "type": "doc",
   "value": {
     "id": "visualization:a42c0580-3224-11e8-a572-ffca06da1357",
-    "index": ".kibana",
+    "index": ".kibana_analytics",
     "source": {
       "coreMigrationVersion": "7.14.0",
       "migrationVersion": {
@@ -102,7 +102,7 @@
   "type": "doc",
   "value": {
     "id": "dashboard:b70c7ae0-3224-11e8-a572-ffca06da1357",
-    "index": ".kibana",
+    "index": ".kibana_analytics",
     "source": {
       "coreMigrationVersion": "7.14.0",
       "dashboard": {