[Fleet] Honor index mode time_serie from the package

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx

@@ -57,17 +57,22 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
     experimentalDataFeatures ?? [],
     registryDataStream
   );
+
+  const isTimeSeriesEnabledByDefault =
+    registryDataStream.elasticsearch?.index_mode === 'time_series';
+
   const isSyntheticSourceEnabledByDefault =
-    registryDataStream.elasticsearch?.source_mode === 'synthetic';
+    registryDataStream.elasticsearch?.source_mode === 'synthetic' || isTimeSeriesEnabledByDefault;
 
   const newExperimentalIndexingFeature = {
     synthetic_source:
       typeof syntheticSourceExperimentalValue !== 'undefined'
         ? syntheticSourceExperimentalValue
         : isSyntheticSourceEnabledByDefault,
-    tsdb:
-      getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ??
-      false,
+    tsdb: isTimeSeriesEnabledByDefault
+      ? isTimeSeriesEnabledByDefault
+      : getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ??
+        false,
   };
 
   const onIndexingSettingChange = (

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx

@@ -19,7 +19,6 @@ import {
   EuiButtonEmpty,
 } from '@elastic/eui';
 
-import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services';
 import type {
   NewPackagePolicy,
   NewPackagePolicyInput,
@@ -127,41 +126,6 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
       [packageInputStreams, packagePolicyInput.streams]
     );
 
-    // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series
-    let isUpdated = false;
-    inputStreams.forEach(({ packagePolicyInputStream }) => {
-      const dataStreamInfo = packageInfo.data_streams?.find(
-        (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset
-      );
-
-      if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') {
-        if (!packagePolicy.package) return;
-        if (!packagePolicy.package?.experimental_data_stream_features)
-          packagePolicy.package!.experimental_data_stream_features = [];
-
-        const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream);
-        const match = packagePolicy.package!.experimental_data_stream_features.find(
-          (feat) => feat.data_stream === dsName
-        );
-        if (match) {
-          if (!match.features.tsdb) {
-            match.features.tsdb = true;
-            isUpdated = true;
-          }
-        } else {
-          packagePolicy.package!.experimental_data_stream_features.push({
-            data_stream: dsName,
-            features: { tsdb: true, synthetic_source: false },
-          });
-          isUpdated = true;
-        }
-      }
-    });
-
-    if (isUpdated) {
-      updatePackagePolicy(packagePolicy);
-    }
-
     return (
       <>
         {/* Header / input-level toggle */}

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts

@@ -6,14 +6,31 @@
  */
 import { createAppContextStartContractMock } from '../../../../mocks';
 import { appContextService } from '../../..';
-
+import { loadFieldsFromYaml } from '../../fields/field';
 import type { RegistryDataStream } from '../../../../types';
 
 import { prepareTemplate } from './install';
 
+jest.mock('../../fields/field', () => ({
+  ...jest.requireActual('../../fields/field'),
+  loadFieldsFromYaml: jest.fn(),
+}));
+
+const mockedLoadFieldsFromYaml = loadFieldsFromYaml as jest.MockedFunction<
+  typeof loadFieldsFromYaml
+>;
+
 describe('EPM index template install', () => {
   beforeEach(async () => {
     appContextService.start(createAppContextStartContractMock());
+
+    mockedLoadFieldsFromYaml.mockReturnValue([
+      {
+        name: 'test_dimension',
+        dimension: true,
+        type: 'keyword',
+      },
+    ]);
   });
 
   it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
@@ -123,6 +140,80 @@ describe('EPM index template install', () => {
     expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' });
   });
 
+  it('tests prepareTemplate to set source mode to synthetics if index_mode:time_series', async () => {
+    const dataStreamDatasetIsPrefixTrue = {
+      type: 'metrics',
+      dataset: 'package.dataset',
+      title: 'test data stream',
+      release: 'experimental',
+      package: 'package',
+      path: 'path',
+      ingest_pipeline: 'default',
+      dataset_is_prefix: true,
+      elasticsearch: {
+        index_mode: 'time_series',
+      },
+    } as RegistryDataStream;
+    const pkg = {
+      name: 'package',
+      version: '0.0.1',
+    };
+
+    const { componentTemplates } = prepareTemplate({
+      pkg,
+      dataStream: dataStreamDatasetIsPrefixTrue,
+    });
+
+    const packageTemplate = componentTemplates['metrics-package.dataset@package'].template;
+
+    if (!('mappings' in packageTemplate)) {
+      throw new Error('no mappings on package template');
+    }
+
+    expect(packageTemplate.mappings).toHaveProperty('_source');
+    expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' });
+  });
+
+  it('tests prepareTemplate to not set source mode to synthetics if index_mode:time_series and user disabled synthetic', async () => {
+    const dataStreamDatasetIsPrefixTrue = {
+      type: 'metrics',
+      dataset: 'package.dataset',
+      title: 'test data stream',
+      release: 'experimental',
+      package: 'package',
+      path: 'path',
+      ingest_pipeline: 'default',
+      dataset_is_prefix: true,
+      elasticsearch: {
+        index_mode: 'time_series',
+      },
+    } as RegistryDataStream;
+    const pkg = {
+      name: 'package',
+      version: '0.0.1',
+    };
+
+    const { componentTemplates } = prepareTemplate({
+      pkg,
+      dataStream: dataStreamDatasetIsPrefixTrue,
+      experimentalDataStreamFeature: {
+        data_stream: 'metrics-package.dataset',
+        features: {
+          synthetic_source: false,
+          tsdb: false,
+        },
+      },
+    });
+
+    const packageTemplate = componentTemplates['metrics-package.dataset@package'].template;
+
+    if (!('mappings' in packageTemplate)) {
+      throw new Error('no mappings on package template');
+    }
+
+    expect(packageTemplate.mappings).not.toHaveProperty('_source');
+  });
+
   it('tests prepareTemplate to not set source mode to synthetics if specified but user disabled it', async () => {
     const dataStreamDatasetIsPrefixTrue = {
       type: 'metrics',
@@ -162,4 +253,33 @@ describe('EPM index template install', () => {
 
     expect(packageTemplate.mappings).not.toHaveProperty('_source');
   });
+
+  it('tests prepareTemplate to set index_mode time series if index_mode:time_series', async () => {
+    const dataStreamDatasetIsPrefixTrue = {
+      type: 'metrics',
+      dataset: 'package.dataset',
+      title: 'test data stream',
+      release: 'experimental',
+      package: 'package',
+      path: 'path',
+      ingest_pipeline: 'default',
+      dataset_is_prefix: true,
+      elasticsearch: {
+        index_mode: 'time_series',
+      },
+    } as RegistryDataStream;
+    const pkg = {
+      name: 'package',
+      version: '0.0.1',
+    };
+
+    const { indexTemplate } = prepareTemplate({
+      pkg,
+      dataStream: dataStreamDatasetIsPrefixTrue,
+    });
+
+    expect(indexTemplate.indexTemplate.template.settings).toEqual({
+      index: { mode: 'time_series', routing_path: ['test_dimension'] },
+    });
+  });
 });

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -281,10 +281,15 @@ export function buildComponentTemplates(params: {
     (dynampingTemplate) => Object.keys(dynampingTemplate)[0]
   );
 
+  const isTimeSeriesEnabledByDefault = registryElasticsearch?.index_mode === 'time_series';
+  const isSyntheticSourceEnabledByDefault = registryElasticsearch?.source_mode === 'synthetic';
+
   const sourceModeSynthetic =
     params.experimentalDataStreamFeature?.features.synthetic_source !== false &&
     (params.experimentalDataStreamFeature?.features.synthetic_source === true ||
-      registryElasticsearch?.source_mode === 'synthetic');
+      isSyntheticSourceEnabledByDefault ||
+      isTimeSeriesEnabledByDefault);
+
   templatesMap[packageTemplateName] = {
     template: {
       settings: {
@@ -517,6 +522,8 @@ export function prepareTemplate({
     composedOfTemplates: Object.keys(componentTemplates),
     templatePriority,
     hidden: dataStream.hidden,
+    registryElasticsearch: dataStream.elasticsearch,
+    mappings,
   });
 
   return {

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -52,6 +52,7 @@ describe('EPM template', () => {
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
+      mappings: { properties: [] },
     });
     expect(template.index_patterns).toStrictEqual([templateIndexPattern]);
   });
@@ -64,6 +65,7 @@ describe('EPM template', () => {
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
+      mappings: { properties: [] },
     });
     expect(template.composed_of).toStrictEqual([
       ...composedOfTemplates,
@@ -79,6 +81,7 @@ describe('EPM template', () => {
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
+      mappings: { properties: [] },
     });
     expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES);
   });
@@ -92,6 +95,7 @@ describe('EPM template', () => {
       composedOfTemplates: [],
       templatePriority: 200,
       hidden: true,
+      mappings: { properties: [] },
     });
     expect(templateWithHidden.data_stream.hidden).toEqual(true);
 
@@ -100,6 +104,7 @@ describe('EPM template', () => {
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
+      mappings: { properties: [] },
     });
     expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
   });

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -14,9 +14,11 @@ import type {
   IndexTemplateEntry,
   IndexTemplate,
   IndexTemplateMappings,
+  RegistryElasticsearch,
 } from '../../../../types';
 import { appContextService } from '../../..';
 import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services';
+import { builRoutingPath } from '../../../package_policies';
 import {
   FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
   FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
@@ -62,20 +64,26 @@ export function getTemplate({
   composedOfTemplates,
   templatePriority,
   hidden,
+  registryElasticsearch,
+  mappings,
 }: {
   templateIndexPattern: string;
   packageName: string;
   composedOfTemplates: string[];
   templatePriority: number;
+  mappings: IndexTemplateMappings;
   hidden?: boolean;
+  registryElasticsearch?: RegistryElasticsearch | undefined;
 }): IndexTemplate {
-  const template = getBaseTemplate(
+  const template = getBaseTemplate({
     templateIndexPattern,
     packageName,
     composedOfTemplates,
     templatePriority,
-    hidden
-  );
+    registryElasticsearch,
+    hidden,
+    mappings,
+  });
   if (template.template.settings.index.final_pipeline) {
     throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`);
   }
@@ -470,21 +478,47 @@ const flattenFieldsToNameAndType = (
   return newFields;
 };
 
-function getBaseTemplate(
-  templateIndexPattern: string,
-  packageName: string,
-  composedOfTemplates: string[],
-  templatePriority: number,
-  hidden?: boolean
-): IndexTemplate {
+function getBaseTemplate({
+  templateIndexPattern,
+  packageName,
+  composedOfTemplates,
+  templatePriority,
+  hidden,
+  registryElasticsearch,
+  mappings,
+}: {
+  templateIndexPattern: string;
+  packageName: string;
+  composedOfTemplates: string[];
+  templatePriority: number;
+  hidden?: boolean;
+  registryElasticsearch: RegistryElasticsearch | undefined;
+  mappings: IndexTemplateMappings;
+}): IndexTemplate {
   const _meta = getESAssetMetadata({ packageName });
 
+  const isIndexModeTimeSeries = registryElasticsearch?.index_mode === 'time_series';
+
+  const mappingsProperties = mappings?.properties ?? {};
+
+  // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path
+  // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592
+  const routingPath = builRoutingPath(mappingsProperties);
+
+  let settingsIndex = {};
+  if (isIndexModeTimeSeries && routingPath.length > 0) {
+    settingsIndex = {
+      mode: 'time_series',
+      routing_path: routingPath,
+    };
+  }
+
   return {
     priority: templatePriority,
     index_patterns: [templateIndexPattern],
     template: {
       settings: {
-        index: {},
+        index: settingsIndex,
       },
       mappings: {
         _meta,

x-pack/test/fleet_api_integration/apis/epm/template.ts

@@ -34,6 +34,7 @@ export default function ({ getService }: FtrProviderContext) {
         packageName: 'system',
         composedOfTemplates: [],
         templatePriority: 200,
+        mappings: { properties: [] },
       });
 
       // This test is not an API integration test with Kibana