[EPM][Security Solution] Implementing dataset component templates

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -175,6 +175,12 @@ export interface Dataset {
  package: string;
  path: string;
  ingest_pipeline: string;
+ elasticsearch?: RegistryElasticsearch;
+}
+
+export interface RegistryElasticsearch {
+ 'index_template.settings'?: object;
+ 'index_template.mappings'?: object;
 }
 
 // EPR types this as `[]map[string]interface{}`
@@ -272,6 +278,7 @@ export interface IndexTemplate {
  data_stream: {
  timestamp_field: string;
  };
+ composed_of: string[];
  _meta: object;
 }
 

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts

@@ -5,7 +5,13 @@
  */
 
 import Boom from 'boom';
-import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types';
+import {
+ Dataset,
+ RegistryPackage,
+ ElasticsearchAssetType,
+ TemplateRef,
+ RegistryElasticsearch,
+} from '../../../../types';
 import { CallESAsCurrentUser } from '../../../../types';
 import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
@@ -157,6 +163,95 @@ export async function installTemplateForDataset({
  });
 }
 
+function putComponentTemplate(
+ body: object | undefined,
+ name: string,
+ callCluster: CallESAsCurrentUser
+): { clusterPromise: Promise<any>; name: string } | undefined {
+ if (body) {
+ const callClusterParams: {
+ method: string;
+ path: string;
+ ignore: number[];
+ body: any;
+ } = {
+ method: 'PUT',
+ path: `/_component_template/${name}`,
+ ignore: [404],
+ body,
+ };
+
+ return { clusterPromise: callCluster('transport.request', callClusterParams), name };
+ }
+}
+
+function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
+ let mappingsTemplate;
+ let settingsTemplate;
+
+ if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
+ mappingsTemplate = {
+ template: {
+ mappings: {
+ ...registryElasticsearch['index_template.mappings'],
+ // temporary change until https://github.com/elastic/elasticsearch/issues/58956 is resolved
+ properties: {
+ '@timestamp': {
+ type: 'date',
+ },
+ },
+ },
+ },
+ };
+ }
+
+ if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
+ settingsTemplate = {
+ template: {
+ settings: registryElasticsearch['index_template.settings'],
+ },
+ };
+ }
+ return { settingsTemplate, mappingsTemplate };
+}
+
+async function installDatasetComponentTemplates(
+ templateName: string,
+ registryElasticsearch: RegistryElasticsearch | undefined,
+ callCluster: CallESAsCurrentUser
+) {
+ const templates: string[] = [];
+ const componentPromises: Array<Promise<any>> = [];
+
+ const compTemplates = buildComponentTemplates(registryElasticsearch);
+
+ const mappings = putComponentTemplate(
+ compTemplates.mappingsTemplate,
+ `${templateName}-mappings`,
+ callCluster
+ );
+
+ const settings = putComponentTemplate(
+ compTemplates.settingsTemplate,
+ `${templateName}-settings`,
+ callCluster
+ );
+
+ if (mappings) {
+ templates.push(mappings.name);
+ componentPromises.push(mappings.clusterPromise);
+ }
+
+ if (settings) {
+ templates.push(settings.name);
+ componentPromises.push(settings.clusterPromise);
+ }
+
+ // TODO: Check return values for errors
+ await Promise.all(componentPromises);
+ return templates;
+}
+
 export async function installTemplate({
  callCluster,
  fields,
@@ -180,13 +275,22 @@ export async function installTemplate({
  packageVersion,
  });
  }
+
+ const composedOfTemplates = await installDatasetComponentTemplates(
+ templateName,
+ dataset.elasticsearch,
+ callCluster
+ );
+
  const template = getTemplate({
  type: dataset.type,
  templateName,
  mappings,
  pipelineName,
  packageName,
+ composedOfTemplates,
  });
+
  // TODO: Check return values for errors
  const callClusterParams: {
  method: string;

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts

@@ -29,10 +29,37 @@ test('get template', () => {
  templateName,
  packageName: 'nginx',
  mappings: { properties: {} },
+ composedOfTemplates: [],
  });
  expect(template.index_patterns).toStrictEqual([`${templateName}-*`]);
 });
 
+test('adds composed_of correctly', () => {
+ const composedOfTemplates = ['component1', 'component2'];
+
+ const template = getTemplate({
+ type: 'logs',
+ templateName: 'name',
+ packageName: 'nginx',
+ mappings: { properties: {} },
+ composedOfTemplates,
+ });
+ expect(template.composed_of).toStrictEqual(composedOfTemplates);
+});
+
+test('adds empty composed_of correctly', () => {
+ const composedOfTemplates: string[] = [];
+
+ const template = getTemplate({
+ type: 'logs',
+ templateName: 'name',
+ packageName: 'nginx',
+ mappings: { properties: {} },
+ composedOfTemplates,
+ });
+ expect(template.composed_of).toStrictEqual(composedOfTemplates);
+});
+
 test('tests loading base.yml', () => {
  const ymlPath = path.join(__dirname, '../../fields/tests/base.yml');
  const fieldsYML = readFileSync(ymlPath, 'utf-8');
@@ -45,6 +72,7 @@ test('tests loading base.yml', () => {
  templateName: 'foo',
  packageName: 'nginx',
  mappings,
+ composedOfTemplates: [],
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -62,6 +90,7 @@ test('tests loading coredns.logs.yml', () => {
  templateName: 'foo',
  packageName: 'coredns',
  mappings,
+ composedOfTemplates: [],
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -79,6 +108,7 @@ test('tests loading system.yml', () => {
  templateName: 'whatsthis',
  packageName: 'system',
  mappings,
+ composedOfTemplates: [],
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -43,14 +43,16 @@ export function getTemplate({
  mappings,
  pipelineName,
  packageName,
+ composedOfTemplates,
 }: {
  type: string;
  templateName: string;
  mappings: IndexTemplateMappings;
  pipelineName?: string | undefined;
  packageName: string;
+ composedOfTemplates: string[];
 }): IndexTemplate {
- const template = getBaseTemplate(type, templateName, mappings, packageName);
+ const template = getBaseTemplate(type, templateName, mappings, packageName, composedOfTemplates);
  if (pipelineName) {
  template.template.settings.index.default_pipeline = pipelineName;
  }
@@ -244,7 +246,8 @@ function getBaseTemplate(
  type: string,
  templateName: string,
  mappings: IndexTemplateMappings,
- packageName: string
+ packageName: string,
+ composedOfTemplates: string[]
 ): IndexTemplate {
  return {
  // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*)
@@ -308,6 +311,7 @@ function getBaseTemplate(
  data_stream: {
  timestamp_field: '@timestamp',
  },
+ composed_of: composedOfTemplates,
  _meta: {
  package: {
  name: packageName,

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -40,6 +40,7 @@ export {
  PackageInfo,
  RegistryVarsEntry,
  Dataset,
+ RegistryElasticsearch,
  AssetReference,
  ElasticsearchAssetType,
  IngestAssetType,

x-pack/test/ingest_manager_api_integration/apis/template.ts

@@ -27,6 +27,7 @@ export default function ({ getService }: FtrProviderContext) {
  templateName,
  mappings,
  packageName: 'system',
+ composedOfTemplates: [],
  });
 
  // This test is not an API integration test with Kibana