diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx index 8910805f52df5..28ce97d17fe97 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx @@ -57,17 +57,22 @@ export const ExperimentDatastreamSettings: React.FunctionComponent = ({ experimentalDataFeatures ?? [], registryDataStream ); + + const isTimeSeriesEnabledByDefault = + registryDataStream.elasticsearch?.index_mode === 'time_series'; + const isSyntheticSourceEnabledByDefault = - registryDataStream.elasticsearch?.source_mode === 'synthetic'; + registryDataStream.elasticsearch?.source_mode === 'synthetic' || isTimeSeriesEnabledByDefault; const newExperimentalIndexingFeature = { synthetic_source: typeof syntheticSourceExperimentalValue !== 'undefined' ? syntheticSourceExperimentalValue : isSyntheticSourceEnabledByDefault, - tsdb: - getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ?? - false, + tsdb: isTimeSeriesEnabledByDefault + ? isTimeSeriesEnabledByDefault + : getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ?? + false, }; const onIndexingSettingChange = ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 8ea01f3100b75..ccb6a6f47e1c8 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -19,7 +19,6 @@ import { EuiButtonEmpty, } from '@elastic/eui'; -import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; import type { NewPackagePolicy, NewPackagePolicyInput, @@ -127,41 +126,6 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ [packageInputStreams, packagePolicyInput.streams] ); - // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series - let isUpdated = false; - inputStreams.forEach(({ packagePolicyInputStream }) => { - const dataStreamInfo = packageInfo.data_streams?.find( - (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset - ); - - if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') { - if (!packagePolicy.package) return; - if (!packagePolicy.package?.experimental_data_stream_features) - packagePolicy.package!.experimental_data_stream_features = []; - - const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream); - const match = packagePolicy.package!.experimental_data_stream_features.find( - (feat) => feat.data_stream === dsName - ); - if (match) { - if (!match.features.tsdb) { - match.features.tsdb = true; - isUpdated = true; - } - } else { - packagePolicy.package!.experimental_data_stream_features.push({ - data_stream: dsName, - features: { tsdb: true, synthetic_source: false }, - }); - isUpdated = true; - } - } - }); - - if (isUpdated) { - updatePackagePolicy(packagePolicy); - } - return ( <> {/* Header / input-level toggle */} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 4b62a2f4a60fb..c293b38a6f46a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -6,14 +6,31 @@ */ import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; - +import { loadFieldsFromYaml } from '../../fields/field'; import type { RegistryDataStream } from '../../../../types'; import { prepareTemplate } from './install'; +jest.mock('../../fields/field', () => ({ + ...jest.requireActual('../../fields/field'), + loadFieldsFromYaml: jest.fn(), +})); + +const mockedLoadFieldsFromYaml = loadFieldsFromYaml as jest.MockedFunction< + typeof loadFieldsFromYaml +>; + describe('EPM index template install', () => { beforeEach(async () => { appContextService.start(createAppContextStartContractMock()); + + mockedLoadFieldsFromYaml.mockReturnValue([ + { + name: 'test_dimension', + dimension: true, + type: 'keyword', + }, + ]); }); it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { @@ -123,6 +140,80 @@ describe('EPM index template install', () => { expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' }); }); + it('tests prepareTemplate to set source mode to synthetics if index_mode:time_series', async () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + elasticsearch: { + index_mode: 'time_series', + }, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + + const { componentTemplates } = prepareTemplate({ + pkg, + dataStream: dataStreamDatasetIsPrefixTrue, + }); + + const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; + + if (!('mappings' in packageTemplate)) { + throw new Error('no mappings on package template'); + } + + expect(packageTemplate.mappings).toHaveProperty('_source'); + expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' }); + }); + + it('tests prepareTemplate to not set source mode to synthetics if index_mode:time_series and user disabled synthetic', async () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + elasticsearch: { + index_mode: 'time_series', + }, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + + const { componentTemplates } = prepareTemplate({ + pkg, + dataStream: dataStreamDatasetIsPrefixTrue, + experimentalDataStreamFeature: { + data_stream: 'metrics-package.dataset', + features: { + synthetic_source: false, + tsdb: false, + }, + }, + }); + + const packageTemplate = componentTemplates['metrics-package.dataset@package'].template; + + if (!('mappings' in packageTemplate)) { + throw new Error('no mappings on package template'); + } + + expect(packageTemplate.mappings).not.toHaveProperty('_source'); + }); + it('tests prepareTemplate to not set source mode to synthetics if specified but user disabled it', async () => { const dataStreamDatasetIsPrefixTrue = { type: 'metrics', @@ -162,4 +253,33 @@ describe('EPM index template install', () => { expect(packageTemplate.mappings).not.toHaveProperty('_source'); }); + + it('tests prepareTemplate to set index_mode time series if index_mode:time_series', async () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + elasticsearch: { + index_mode: 'time_series', + }, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + + const { indexTemplate } = prepareTemplate({ + pkg, + dataStream: dataStreamDatasetIsPrefixTrue, + }); + + expect(indexTemplate.indexTemplate.template.settings).toEqual({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index df744c7ae532e..d124d1bbfb643 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -281,10 +281,15 @@ export function buildComponentTemplates(params: { (dynampingTemplate) => Object.keys(dynampingTemplate)[0] ); + const isTimeSeriesEnabledByDefault = registryElasticsearch?.index_mode === 'time_series'; + const isSyntheticSourceEnabledByDefault = registryElasticsearch?.source_mode === 'synthetic'; + const sourceModeSynthetic = params.experimentalDataStreamFeature?.features.synthetic_source !== false && (params.experimentalDataStreamFeature?.features.synthetic_source === true || - registryElasticsearch?.source_mode === 'synthetic'); + isSyntheticSourceEnabledByDefault || + isTimeSeriesEnabledByDefault); + templatesMap[packageTemplateName] = { template: { settings: { @@ -517,6 +522,8 @@ export function prepareTemplate({ composedOfTemplates: Object.keys(componentTemplates), templatePriority, hidden: dataStream.hidden, + registryElasticsearch: dataStream.elasticsearch, + mappings, }); return { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 0d26984c55be5..d6af8890686f8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -52,6 +52,7 @@ describe('EPM template', () => { packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, + mappings: { properties: [] }, }); expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); @@ -64,6 +65,7 @@ describe('EPM template', () => { packageName: 'nginx', composedOfTemplates, templatePriority: 200, + mappings: { properties: [] }, }); expect(template.composed_of).toStrictEqual([ ...composedOfTemplates, @@ -79,6 +81,7 @@ describe('EPM template', () => { packageName: 'nginx', composedOfTemplates, templatePriority: 200, + mappings: { properties: [] }, }); expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES); }); @@ -92,6 +95,7 @@ describe('EPM template', () => { composedOfTemplates: [], templatePriority: 200, hidden: true, + mappings: { properties: [] }, }); expect(templateWithHidden.data_stream.hidden).toEqual(true); @@ -100,6 +104,7 @@ describe('EPM template', () => { packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, + mappings: { properties: [] }, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index c0187e1446258..23c55db1d43f0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -14,9 +14,11 @@ import type { IndexTemplateEntry, IndexTemplate, IndexTemplateMappings, + RegistryElasticsearch, } from '../../../../types'; import { appContextService } from '../../..'; import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services'; +import { builRoutingPath } from '../../../package_policies'; import { FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, @@ -62,20 +64,26 @@ export function getTemplate({ composedOfTemplates, templatePriority, hidden, + registryElasticsearch, + mappings, }: { templateIndexPattern: string; packageName: string; composedOfTemplates: string[]; templatePriority: number; + mappings: IndexTemplateMappings; hidden?: boolean; + registryElasticsearch?: RegistryElasticsearch | undefined; }): IndexTemplate { - const template = getBaseTemplate( + const template = getBaseTemplate({ templateIndexPattern, packageName, composedOfTemplates, templatePriority, - hidden - ); + registryElasticsearch, + hidden, + mappings, + }); if (template.template.settings.index.final_pipeline) { throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } @@ -470,21 +478,47 @@ const flattenFieldsToNameAndType = ( return newFields; }; -function getBaseTemplate( - templateIndexPattern: string, - packageName: string, - composedOfTemplates: string[], - templatePriority: number, - hidden?: boolean -): IndexTemplate { +function getBaseTemplate({ + templateIndexPattern, + packageName, + composedOfTemplates, + templatePriority, + hidden, + registryElasticsearch, + mappings, +}: { + templateIndexPattern: string; + packageName: string; + composedOfTemplates: string[]; + templatePriority: number; + hidden?: boolean; + registryElasticsearch: RegistryElasticsearch | undefined; + mappings: IndexTemplateMappings; +}): IndexTemplate { const _meta = getESAssetMetadata({ packageName }); + const isIndexModeTimeSeries = registryElasticsearch?.index_mode === 'time_series'; + + const mappingsProperties = mappings?.properties ?? {}; + + // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path + // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592 + const routingPath = builRoutingPath(mappingsProperties); + + let settingsIndex = {}; + if (isIndexModeTimeSeries && routingPath.length > 0) { + settingsIndex = { + mode: 'time_series', + routing_path: routingPath, + }; + } + return { priority: templatePriority, index_patterns: [templateIndexPattern], template: { settings: { - index: {}, + index: settingsIndex, }, mappings: { _meta, diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index 30b2baebe959b..c62afa6dccbf7 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -34,6 +34,7 @@ export default function ({ getService }: FtrProviderContext) { packageName: 'system', composedOfTemplates: [], templatePriority: 200, + mappings: { properties: [] }, }); // This test is not an API integration test with Kibana