[Observability][SecuritySolution] Update entity manager to support extension of mappings and ingest pipeline

x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts

@@ -18,8 +18,6 @@ export const ENTITY_EVENT_COMPONENT_TEMPLATE_V1 =
 
 // History constants
 export const ENTITY_HISTORY = 'history' as const;
-export const ENTITY_HISTORY_INDEX_TEMPLATE_V1 =
-  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_index_template` as const;
 export const ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_base` as const;
 export const ENTITY_HISTORY_PREFIX_V1 =
@@ -29,8 +27,6 @@ export const ENTITY_HISTORY_INDEX_PREFIX_V1 =
 
 // Latest constants
 export const ENTITY_LATEST = 'latest' as const;
-export const ENTITY_LATEST_INDEX_TEMPLATE_V1 =
-  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_index_template` as const;
 export const ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_base` as const;
 export const ENTITY_LATEST_PREFIX_V1 =

x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEntityHistoryIndexTemplateV1, getEntityLatestIndexTemplateV1 } from './helpers';
+
+describe('helpers', () => {
+  it('getEntityHistoryIndexTemplateV1 should return the correct value', () => {
+    const definitionId = 'test';
+    const result = getEntityHistoryIndexTemplateV1(definitionId);
+    expect(result).toEqual('entities_v1_history_test_index_template');
+  });
+
+  it('getEntityLatestIndexTemplateV1 should return the correct value', () => {
+    const definitionId = 'test';
+    const result = getEntityLatestIndexTemplateV1(definitionId);
+    expect(result).toEqual('entities_v1_latest_test_index_template');
+  });
+});

x-pack/plugins/observability_solution/entity_manager/common/helpers.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ENTITY_BASE_PREFIX,
+  ENTITY_HISTORY,
+  ENTITY_LATEST,
+  ENTITY_SCHEMA_VERSION_V1,
+} from './constants_entities';
+
+export const getEntityHistoryIndexTemplateV1 = (definitionId: string) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definitionId}_index_template` as const;
+
+export const getEntityLatestIndexTemplateV1 = (definitionId: string) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definitionId}_index_template` as const;

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts

@@ -163,5 +163,30 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
         date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"],
       },
     },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-history@platform`,
+      },
+    },
+
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@custom`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-history@custom`,
+      },
+    },
   ];
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts

@@ -122,5 +122,30 @@ export function generateLatestProcessors(definition: EntityDefinition) {
         value: `${generateLatestIndexName(definition)}`,
       },
     },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-latest@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@custom`,
+      },
+    },
+
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-latest@custom`,
+      },
+    },
   ];
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts

@@ -34,6 +34,18 @@ const assertHasCreatedDefinition = (
     overwrite: true,
   });
 
+  expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
+  expect(esClient.indices.putIndexTemplate).toBeCalledWith(
+    expect.objectContaining({
+      name: `entities_v1_history_${definition.id}_index_template`,
+    })
+  );
+  expect(esClient.indices.putIndexTemplate).toBeCalledWith(
+    expect.objectContaining({
+      name: `entities_v1_latest_${definition.id}_index_template`,
+    })
+  );
+
   expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition),
@@ -111,6 +123,14 @@ const assertHasUninstalledDefinition = (
   expect(esClient.transform.deleteTransform).toBeCalledTimes(2);
   expect(esClient.ingest.deletePipeline).toBeCalledTimes(2);
   expect(soClient.delete).toBeCalledTimes(1);
+
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2);
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({
+    name: `entities_v1_history_${definition.id}_index_template`,
+  });
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledWith({
+    name: `entities_v1_latest_${definition.id}_index_template`,
+  });
 };
 
 describe('install_entity_definition', () => {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts

@@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
+import {
+  getEntityHistoryIndexTemplateV1,
+  getEntityLatestIndexTemplateV1,
+} from '../../../common/helpers';
 import {
   createAndInstallHistoryIngestPipeline,
   createAndInstallLatestIngestPipeline,
@@ -28,6 +32,9 @@ import {
   stopAndDeleteLatestTransform,
 } from './stop_and_delete_transform';
 import { uninstallEntityDefinition } from './uninstall_entity_definition';
+import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
+import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template';
+import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template';
 
 export interface InstallDefinitionParams {
   esClient: ElasticsearchClient;
@@ -52,6 +59,10 @@ export async function installEntityDefinition({
       latest: false,
     },
     definition: false,
+    indexTemplates: {
+      history: false,
+      latest: false,
+    },
   };
 
   try {
@@ -62,6 +73,20 @@ export async function installEntityDefinition({
     const entityDefinition = await saveEntityDefinition(soClient, definition);
     installState.definition = true;
 
+    // install scoped index template
+    await upsertTemplate({
+      esClient,
+      logger,
+      template: getEntitiesHistoryIndexTemplateConfig(definition.id),
+    });
+    installState.indexTemplates.history = true;
+    await upsertTemplate({
+      esClient,
+      logger,
+      template: getEntitiesLatestIndexTemplateConfig(definition.id),
+    });
+    installState.indexTemplates.latest = true;
+
     // install ingest pipelines
     logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
     await createAndInstallHistoryIngestPipeline(esClient, entityDefinition, logger);
@@ -84,6 +109,21 @@ export async function installEntityDefinition({
       await deleteEntityDefinition(soClient, definition, logger);
     }
 
+    if (installState.indexTemplates.history) {
+      await deleteTemplate({
+        esClient,
+        logger,
+        name: getEntityHistoryIndexTemplateV1(definition.id),
+      });
+    }
+    if (installState.indexTemplates.latest) {
+      await deleteTemplate({
+        esClient,
+        logger,
+        name: getEntityLatestIndexTemplateV1(definition.id),
+      });
+    }
+
     if (installState.ingestPipelines.history) {
       await deleteHistoryIngestPipeline(esClient, definition, logger);
     }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts

@@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
+import {
+  getEntityHistoryIndexTemplateV1,
+  getEntityLatestIndexTemplateV1,
+} from '../../../common/helpers';
 import { deleteEntityDefinition } from './delete_entity_definition';
 import { deleteIndices } from './delete_index';
 import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
@@ -17,6 +21,7 @@ import {
   stopAndDeleteHistoryTransform,
   stopAndDeleteLatestTransform,
 } from './stop_and_delete_transform';
+import { deleteTemplate } from '../manage_index_templates';
 
 export async function uninstallEntityDefinition({
   definition,
@@ -36,6 +41,9 @@ export async function uninstallEntityDefinition({
   await deleteHistoryIngestPipeline(esClient, definition, logger);
   await deleteLatestIngestPipeline(esClient, definition, logger);
   await deleteEntityDefinition(soClient, definition, logger);
+  await deleteTemplate({ esClient, logger, name: getEntityHistoryIndexTemplateV1(definition.id) });
+  await deleteTemplate({ esClient, logger, name: getEntityLatestIndexTemplateV1(definition.id) });
+
   if (deleteData) {
     await deleteIndices(esClient, definition, logger);
   }

x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts

@@ -23,6 +23,12 @@ interface ComponentManagementOptions {
   logger: Logger;
 }
 
+interface DeleteTemplateOptions {
+  esClient: ElasticsearchClient;
+  name: string;
+  logger: Logger;
+}
+
 export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) {
   try {
     await esClient.indices.putIndexTemplate(template);
@@ -37,6 +43,15 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan
   logger.debug(() => `Entity manager index template: ${JSON.stringify(template)}`);
 }
 
+export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) {
+  try {
+    await esClient.indices.deleteIndexTemplate({ name });
+  } catch (error: any) {
+    logger.error(`Error deleting entity manager index template: ${error.message}`);
+    return;
+  }
+}
+
 export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) {
   try {
     await esClient.cluster.putComponentTemplate(component);

x-pack/plugins/observability_solution/entity_manager/server/plugin.ts

@@ -14,7 +14,7 @@ import {
   PluginConfigDescriptor,
   Logger,
 } from '@kbn/core/server';
-import { upsertComponent, upsertTemplate } from './lib/manage_index_templates';
+import { upsertComponent } from './lib/manage_index_templates';
 import { setupRoutes } from './routes';
 import {
   EntityManagerPluginSetupDependencies,
@@ -27,8 +27,6 @@ import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects';
 import { entitiesEntityComponentTemplateConfig } from './templates/components/entity';
 import { entitiesLatestBaseComponentTemplateConfig } from './templates/components/base_latest';
 import { entitiesHistoryBaseComponentTemplateConfig } from './templates/components/base_history';
-import { entitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
-import { entitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
 
 export type EntityManagerServerPluginSetup = ReturnType<EntityManagerServerPlugin['setup']>;
 export type EntityManagerServerPluginStart = ReturnType<EntityManagerServerPlugin['start']>;
@@ -113,22 +111,7 @@ export class EntityManagerServerPlugin
         logger: this.logger,
         component: entitiesEntityComponentTemplateConfig,
       }),
-    ])
-      .then(() =>
-        upsertTemplate({
-          esClient,
-          logger: this.logger,
-          template: entitiesHistoryIndexTemplateConfig,
-        })
-      )
-      .then(() =>
-        upsertTemplate({
-          esClient,
-          logger: this.logger,
-          template: entitiesLatestIndexTemplateConfig,
-        })
-      )
-      .catch(() => {});
+    ]).catch(() => {});
 
     return {};
   }