[eem] narrow down index patterns in definition templates (elastic#189182

)

In elastic#188410 we moved history and
latest index templates from global scope to definition scope. The
definition-scoped templates have a wide pattern that would grep any
other definition template already installed and throw the following
error because of conflicting priority. This change narrows down the
index patterns defined in the templates to only grep the ones from the
installed definition

```
{
  "statusCode": 500,
  "error": "Internal Server Error",
  "message": """[illegal_argument_exception
	Root causes:
		illegal_argument_exception: index template [entities_v1_history_admin-console-services_index_template] has index patterns [.entities.v1.history.*] matching patterns from existing templates [entities_v1_history_builtin_services_from_ecs_data_index_template] with patterns (entities_v1_history_builtin_services_from_ecs_data_index_template => [.entities.v1.history.*]) that have the same priority [200], multiple index templates may not match during index creation, please use a different priority]: index template [entities_v1_history_admin-console-services_index_template] has index patterns [.entities.v1.history.*] matching patterns from existing templates [entities_v1_history_builtin_services_from_ecs_data_index_template] with patterns (entities_v1_history_builtin_services_from_ecs_data_index_template => [.entities.v1.history.*]) that have the same priority [200], multiple index templates may not match during index creation, please use a different priority"""
}
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_ingest_pipeline.ts

@@ -24,7 +24,7 @@ export async function deleteHistoryIngestPipeline(
       esClient.ingest.deletePipeline({ id: historyPipelineId }, { ignore: [404] })
     );
   } catch (e) {
-    logger.error(`Unable to delete history ingest pipeline [${definition.id}].`);
+    logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`);
     throw e;
   }
 }
@@ -40,7 +40,7 @@ export async function deleteLatestIngestPipeline(
       esClient.ingest.deletePipeline({ id: latestPipelineId }, { ignore: [404] })
     );
   } catch (e) {
-    logger.error(`Unable to delete latest ingest pipeline [${definition.id}].`);
+    logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`);
     throw e;
   }
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts

@@ -7,7 +7,7 @@
 
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({
-  id: 'admin-console-services',
+  id: 'admin-console-services-backfill',
   version: '999.999.999',
   name: 'Services for Admin Console',
   type: 'service',

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { entityDefinition } from './entity_definition';
+export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts

@@ -36,8 +36,8 @@ import {
 import { uninstallEntityDefinition } from './uninstall_entity_definition';
 import { isBackfillEnabled } from './helpers/is_backfill_enabled';
 import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
-import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template';
-import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template';
+import { getEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
+import { getEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
 
 export interface InstallDefinitionParams {
   esClient: ElasticsearchClient;
@@ -111,7 +111,7 @@ export async function installEntityDefinition({
 
     return entityDefinition;
   } catch (e) {
-    logger.error(`Failed to install entity definition ${definition.id}`, e);
+    logger.error(`Failed to install entity definition ${definition.id}: ${e}`);
     // Clean up anything that was successful.
     if (installState.definition) {
       await deleteEntityDefinition(soClient, definition, logger);

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/read_entity_definition.ts

@@ -30,7 +30,7 @@ export async function readEntityDefinition(
   try {
     return entityDefinitionSchema.parse(response.saved_objects[0].attributes);
   } catch (e) {
-    logger.error(`Unable to parse entity definition with [${id}]`);
+    logger.error(`Unable to parse entity definition with [${id}]: ${e}`);
     throw e;
   }
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts

@@ -38,7 +38,7 @@ export async function stopAndDeleteHistoryTransform(
       { logger }
     );
   } catch (e) {
-    logger.error(`Cannot stop or delete history transform [${definition.id}]`);
+    logger.error(`Cannot stop or delete history transform [${definition.id}]: ${e}`);
     throw e;
   }
 }
@@ -67,7 +67,7 @@ export async function stopAndDeleteHistoryBackfillTransform(
       { logger }
     );
   } catch (e) {
-    logger.error(`Cannot stop or delete history backfill transform [${definition.id}]`);
+    logger.error(`Cannot stop or delete history backfill transform [${definition.id}]: ${e}`);
     throw e;
   }
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entityDefinition } from '../helpers/fixtures/entity_definition';
+import { getEntitiesHistoryIndexTemplateConfig } from './entities_history_template';
+
+describe('getEntitiesHistoryIndexTemplateConfig(definitionId)', () => {
+  it('should generate a valid index template', () => {
+    const template = getEntitiesHistoryIndexTemplateConfig(entityDefinition.id);
+    expect(template).toMatchSnapshot();
+  });
+});

x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts → x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts

@@ -6,14 +6,14 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
-import { getEntityHistoryIndexTemplateV1 } from '../../common/helpers';
+import { getEntityHistoryIndexTemplateV1 } from '../../../../common/helpers';
 import {
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
   ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
   ENTITY_HISTORY_INDEX_PREFIX_V1,
-} from '../../common/constants_entities';
-import { getCustomHistoryTemplateComponents } from './components/helpers';
+} from '../../../../common/constants_entities';
+import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers';
 
 export const getEntitiesHistoryIndexTemplateConfig = (
   definitionId: string
@@ -33,7 +33,7 @@ export const getEntitiesHistoryIndexTemplateConfig = (
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
     ...getCustomHistoryTemplateComponents(definitionId),
   ],
-  index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.*`],
+  index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definitionId}.*`],
   priority: 200,
   template: {
     mappings: {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entityDefinition } from '../helpers/fixtures/entity_definition';
+import { getEntitiesLatestIndexTemplateConfig } from './entities_latest_template';
+
+describe('getEntitiesLatestIndexTemplateConfig(definitionId)', () => {
+  it('should generate a valid index template', () => {
+    const template = getEntitiesLatestIndexTemplateConfig(entityDefinition.id);
+    expect(template).toMatchSnapshot();
+  });
+});

x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts → x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts

@@ -6,14 +6,14 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
-import { getEntityLatestIndexTemplateV1 } from '../../common/helpers';
+import { getEntityLatestIndexTemplateV1 } from '../../../../common/helpers';
 import {
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
   ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
   ENTITY_LATEST_INDEX_PREFIX_V1,
-} from '../../common/constants_entities';
-import { getCustomLatestTemplateComponents } from './components/helpers';
+} from '../../../../common/constants_entities';
+import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers';
 
 export const getEntitiesLatestIndexTemplateConfig = (
   definitionId: string
@@ -33,8 +33,8 @@ export const getEntitiesLatestIndexTemplateConfig = (
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
     ...getCustomLatestTemplateComponents(definitionId),
   ],
-  index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.*`],
-  priority: 1,
+  index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.${definitionId}`],
+  priority: 200,
   template: {
     mappings: {
       _meta: {

x-pack/test/api_integration/apis/entity_manager/definitions.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { EntityDefinition } from '@kbn/entities-schema';
+import {
+  entityDefinition as mockDefinition,
+  entityDefinitionWithBackfill as mockBackfillDefinition,
+} from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { installDefinition, uninstallDefinition, getInstalledDefinitions } from './helpers/request';
+
+export default function ({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+
+  describe('Entity definitions', () => {
+    describe('definitions installations', () => {
+      it('can install multiple definitions', async () => {
+        await installDefinition(supertest, mockDefinition);
+        await installDefinition(supertest, mockBackfillDefinition);
+
+        const { definitions } = await getInstalledDefinitions(supertest);
+        expect(definitions.length).to.eql(2);
+        expect(
+          definitions.find((definition: EntityDefinition) => definition.id === mockDefinition.id)
+        );
+        expect(
+          definitions.find(
+            (definition: EntityDefinition) => definition.id === mockBackfillDefinition.id
+          )
+        );
+
+        await uninstallDefinition(supertest, mockDefinition.id);
+        await uninstallDefinition(supertest, mockBackfillDefinition.id);
+      });
+    });
+  });
+}