[APM] Add experimental support for Data Streams (#89650)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/apm_oss/server/index.ts

@@ -21,6 +21,7 @@ export const config = {
     sourcemapIndices: schema.string({ defaultValue: 'apm-*' }),
     onboardingIndices: schema.string({ defaultValue: 'apm-*' }),
     indexPattern: schema.string({ defaultValue: 'apm-*' }),
+    fleetMode: schema.boolean({ defaultValue: true }),
   }),
 };
 

x-pack/plugins/apm/common/processor_event.ts

@@ -10,8 +10,6 @@ export enum ProcessorEvent {
   error = 'error',
   metric = 'metric',
   span = 'span',
-  onboarding = 'onboarding',
-  sourcemap = 'sourcemap',
 }
 /**
  * Processor events that are searchable in the UI via the query bar.

x-pack/plugins/apm/public/components/shared/KueryBar/index.tsx

@@ -66,7 +66,7 @@ export function KueryBar(props: { prepend?: React.ReactNode | string }) {
 
   const example = examples[processorEvent || 'defaults'];
 
-  const { indexPattern } = useDynamicIndexPatternFetcher(processorEvent);
+  const { indexPattern } = useDynamicIndexPatternFetcher();
 
   const placeholder = i18n.translate('xpack.apm.kueryBar.placeholder', {
     defaultMessage: `Search {event, select,

x-pack/plugins/apm/public/hooks/use_dynamic_index_pattern.ts

@@ -6,25 +6,14 @@
  */
 
 import { useFetcher } from './use_fetcher';
-import { UIProcessorEvent } from '../../common/processor_event';
 
-export function useDynamicIndexPatternFetcher(
-  processorEvent: UIProcessorEvent | undefined
-) {
-  const { data, status } = useFetcher(
-    (callApmApi) => {
-      return callApmApi({
-        endpoint: 'GET /api/apm/index_pattern/dynamic',
-        isCachable: true,
-        params: {
-          query: {
-            processorEvent,
-          },
-        },
-      });
-    },
-    [processorEvent]
-  );
+export function useDynamicIndexPatternFetcher() {
+  const { data, status } = useFetcher((callApmApi) => {
+    return callApmApi({
+      endpoint: 'GET /api/apm/index_pattern/dynamic',
+      isCachable: true,
+    });
+  }, []);
 
   return {
     indexPattern: data?.dynamicIndexPattern,

x-pack/plugins/apm/server/index.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable @typescript-eslint/naming-convention */
+
+import { APMOSSConfig } from 'src/plugins/apm_oss/server';
+import { APMXPackConfig } from '.';
+import { mergeConfigs } from './index';
+
+describe('mergeConfigs', () => {
+  it('merges the configs', () => {
+    const apmOssConfig = {
+      transactionIndices: 'apm-*-transaction-*',
+      spanIndices: 'apm-*-span-*',
+      errorIndices: 'apm-*-error-*',
+      metricsIndices: 'apm-*-metric-*',
+      indexPattern: 'apm-*',
+    } as APMOSSConfig;
+
+    const apmConfig = {
+      ui: { enabled: false },
+      enabled: true,
+      metricsInterval: 2000,
+    } as APMXPackConfig;
+
+    expect(mergeConfigs(apmOssConfig, apmConfig)).toEqual({
+      'apm_oss.errorIndices': 'apm-*-error-*',
+      'apm_oss.indexPattern': 'apm-*',
+      'apm_oss.metricsIndices': 'apm-*-metric-*',
+      'apm_oss.spanIndices': 'apm-*-span-*',
+      'apm_oss.transactionIndices': 'apm-*-transaction-*',
+      'xpack.apm.metricsInterval': 2000,
+      'xpack.apm.ui.enabled': false,
+    });
+  });
+
+  it('adds fleet indices', () => {
+    const apmOssConfig = {
+      transactionIndices: 'apm-*-transaction-*',
+      spanIndices: 'apm-*-span-*',
+      errorIndices: 'apm-*-error-*',
+      metricsIndices: 'apm-*-metric-*',
+      fleetMode: true,
+    } as APMOSSConfig;
+
+    const apmConfig = { ui: {} } as APMXPackConfig;
+
+    expect(mergeConfigs(apmOssConfig, apmConfig)).toEqual({
+      'apm_oss.errorIndices': 'logs-apm*,apm-*-error-*',
+      'apm_oss.metricsIndices': 'metrics-apm*,apm-*-metric-*',
+      'apm_oss.spanIndices': 'traces-apm*,apm-*-span-*',
+      'apm_oss.transactionIndices': 'traces-apm*,apm-*-transaction-*',
+    });
+  });
+
+  it('does not add fleet indices', () => {
+    const apmOssConfig = {
+      transactionIndices: 'apm-*-transaction-*',
+      spanIndices: 'apm-*-span-*',
+      errorIndices: 'apm-*-error-*',
+      metricsIndices: 'apm-*-metric-*',
+      fleetMode: false,
+    } as APMOSSConfig;
+
+    const apmConfig = { ui: {} } as APMXPackConfig;
+
+    expect(mergeConfigs(apmOssConfig, apmConfig)).toEqual({
+      'apm_oss.errorIndices': 'apm-*-error-*',
+      'apm_oss.metricsIndices': 'apm-*-metric-*',
+      'apm_oss.spanIndices': 'apm-*-span-*',
+      'apm_oss.transactionIndices': 'apm-*-transaction-*',
+    });
+  });
+});

x-pack/plugins/apm/server/index.ts

@@ -11,6 +11,7 @@ import { APMOSSConfig } from 'src/plugins/apm_oss/server';
 import { APMPlugin } from './plugin';
 import { SearchAggregatedTransactionSetting } from '../common/aggregated_transactions';
 
+// plugin config
 export const config = {
   exposeToBrowser: {
     serviceMapEnabled: true,
@@ -50,20 +51,23 @@ export const config = {
 };
 
 export type APMXPackConfig = TypeOf<typeof config.schema>;
+export type APMConfig = ReturnType<typeof mergeConfigs>;
 
+// plugin config and ui indices settings
 export function mergeConfigs(
   apmOssConfig: APMOSSConfig,
   apmConfig: APMXPackConfig
 ) {
-  return {
+  const mergedConfig = {
     /* eslint-disable @typescript-eslint/naming-convention */
+    // TODO: Remove all apm_oss options by 8.0
     'apm_oss.transactionIndices': apmOssConfig.transactionIndices,
     'apm_oss.spanIndices': apmOssConfig.spanIndices,
     'apm_oss.errorIndices': apmOssConfig.errorIndices,
     'apm_oss.metricsIndices': apmOssConfig.metricsIndices,
     'apm_oss.sourcemapIndices': apmOssConfig.sourcemapIndices,
     'apm_oss.onboardingIndices': apmOssConfig.onboardingIndices,
-    'apm_oss.indexPattern': apmOssConfig.indexPattern,
+    'apm_oss.indexPattern': apmOssConfig.indexPattern, // TODO: add data stream indices: traces-apm*,logs-apm*,metrics-apm*. Blocked by https://github.com/elastic/kibana/issues/87851
     /* eslint-enable @typescript-eslint/naming-convention */
     'xpack.apm.serviceMapEnabled': apmConfig.serviceMapEnabled,
     'xpack.apm.serviceMapFingerprintBucketSize':
@@ -89,9 +93,27 @@ export function mergeConfigs(
       apmConfig.searchAggregatedTransactions,
     'xpack.apm.metricsInterval': apmConfig.metricsInterval,
   };
-}
 
-export type APMConfig = ReturnType<typeof mergeConfigs>;
+  if (apmOssConfig.fleetMode) {
+    mergedConfig[
+      'apm_oss.transactionIndices'
+    ] = `traces-apm*,${mergedConfig['apm_oss.transactionIndices']}`;
+
+    mergedConfig[
+      'apm_oss.spanIndices'
+    ] = `traces-apm*,${mergedConfig['apm_oss.spanIndices']}`;
+
+    mergedConfig[
+      'apm_oss.errorIndices'
+    ] = `logs-apm*,${mergedConfig['apm_oss.errorIndices']}`;
+
+    mergedConfig[
+      'apm_oss.metricsIndices'
+    ] = `metrics-apm*,${mergedConfig['apm_oss.metricsIndices']}`;
+  }
+
+  return mergedConfig;
+}
 
 export const plugin = (initContext: PluginInitializerContext) =>
   new APMPlugin(initContext);

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -41,8 +41,6 @@ type TypeOfProcessorEvent<T extends ProcessorEvent> = {
   [ProcessorEvent.transaction]: Transaction;
   [ProcessorEvent.span]: Span;
   [ProcessorEvent.metric]: Metric;
-  [ProcessorEvent.onboarding]: unknown;
-  [ProcessorEvent.sourcemap]: unknown;
 }[T];
 
 type ESSearchRequestOf<TParams extends APMEventESSearchRequest> = Omit<

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable @typescript-eslint/naming-convention */
+import { APMEventESSearchRequest } from '.';
+import { ApmIndicesConfig } from '../../../settings/apm_indices/get_apm_indices';
+import { unpackProcessorEvents } from './unpack_processor_events';
+
+describe('unpackProcessorEvents', () => {
+  let res: ReturnType<typeof unpackProcessorEvents>;
+  beforeEach(() => {
+    const request = {
+      apm: { events: ['transaction', 'error'] },
+      body: { query: { bool: { filter: [{ terms: { foo: 'bar' } }] } } },
+    } as APMEventESSearchRequest;
+
+    const indices = {
+      'apm_oss.transactionIndices': 'my-apm-*-transaction-*',
+      'apm_oss.metricsIndices': 'my-apm-*-metric-*',
+      'apm_oss.errorIndices': 'my-apm-*-error-*',
+      'apm_oss.spanIndices': 'my-apm-*-span-*',
+      'apm_oss.onboardingIndices': 'my-apm-*-onboarding-',
+      'apm_oss.sourcemapIndices': 'my-apm-*-sourcemap-*',
+    } as ApmIndicesConfig;
+
+    res = unpackProcessorEvents(request, indices);
+  });
+
+  it('adds terms filter for apm events', () => {
+    expect(res.body.query.bool.filter).toContainEqual({
+      terms: { 'processor.event': ['transaction', 'error'] },
+    });
+  });
+
+  it('merges queries', () => {
+    expect(res.body.query.bool.filter).toEqual([
+      { terms: { foo: 'bar' } },
+      { terms: { 'processor.event': ['transaction', 'error'] } },
+    ]);
+  });
+
+  it('searches the specified indices', () => {
+    expect(res.index).toEqual(['my-apm-*-transaction-*', 'my-apm-*-error-*']);
+  });
+});

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.ts

@@ -18,23 +18,19 @@ import {
   ApmIndicesName,
 } from '../../../settings/apm_indices/get_apm_indices';
 
-export const processorEventIndexMap: Record<ProcessorEvent, ApmIndicesName> = {
+const processorEventIndexMap: Record<ProcessorEvent, ApmIndicesName> = {
   [ProcessorEvent.transaction]: 'apm_oss.transactionIndices',
   [ProcessorEvent.span]: 'apm_oss.spanIndices',
   [ProcessorEvent.metric]: 'apm_oss.metricsIndices',
   [ProcessorEvent.error]: 'apm_oss.errorIndices',
-  [ProcessorEvent.sourcemap]: 'apm_oss.sourcemapIndices',
-  [ProcessorEvent.onboarding]: 'apm_oss.onboardingIndices',
 };
 
 export function unpackProcessorEvents(
   request: APMEventESSearchRequest,
   indices: ApmIndicesConfig
 ) {
   const { apm, ...params } = request;
-
   const events = uniq(apm.events);
-
   const index = events.map((event) => indices[processorEventIndexMap[event]]);
 
   const withFilterForProcessorEvent: ESSearchRequest & {

x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.ts

@@ -14,6 +14,7 @@ import { hasHistoricalAgentData } from '../services/get_services/has_historical_
 import { Setup } from '../helpers/setup_request';
 import { APMRequestHandlerContext } from '../../routes/typings';
 import { InternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client.js';
+import { getApmIndexPatternTitle } from './get_apm_index_pattern_title';
 
 export async function createStaticIndexPattern(
   setup: Setup,
@@ -35,7 +36,7 @@ export async function createStaticIndexPattern(
   }
 
   try {
-    const apmIndexPatternTitle = config['apm_oss.indexPattern'];
+    const apmIndexPatternTitle = getApmIndexPatternTitle(context);
     await savedObjectsClient.create(
       'index-pattern',
       {

x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts

@@ -10,11 +10,6 @@ import {
   IndexPatternsFetcher,
   FieldDescriptor,
 } from '../../../../../../src/plugins/data/server';
-import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices';
-import {
-  ProcessorEvent,
-  UIProcessorEvent,
-} from '../../../common/processor_event';
 import { APMRequestHandlerContext } from '../../routes/typings';
 
 interface IndexPatternTitleAndFields {
@@ -30,15 +25,11 @@ const cache = new LRU<string, IndexPatternTitleAndFields | undefined>({
 // TODO: this is currently cached globally. In the future we might want to cache this per user
 export const getDynamicIndexPattern = async ({
   context,
-  indices,
-  processorEvent,
 }: {
   context: APMRequestHandlerContext;
-  indices: ApmIndicesConfig;
-  processorEvent?: UIProcessorEvent;
 }) => {
-  const patternIndices = getPatternIndices(indices, processorEvent);
-  const indexPatternTitle = patternIndices.join(',');
+  const indexPatternTitle = context.config['apm_oss.indexPattern'];
+
   const CACHE_KEY = `apm_dynamic_index_pattern_${indexPatternTitle}`;
   if (cache.has(CACHE_KEY)) {
     return cache.get(CACHE_KEY);
@@ -54,7 +45,7 @@ export const getDynamicIndexPattern = async ({
   // (would be a bad first time experience)
   try {
     const fields = await indexPatternsFetcher.getFieldsForWildcard({
-      pattern: patternIndices,
+      pattern: indexPatternTitle,
     });
 
     const indexPattern: IndexPatternTitleAndFields = {
@@ -79,20 +70,3 @@ export const getDynamicIndexPattern = async ({
     throw e;
   }
 };
-
-function getPatternIndices(
-  indices: ApmIndicesConfig,
-  processorEvent?: UIProcessorEvent
-) {
-  const indexNames = processorEvent
-    ? [processorEvent]
-    : [ProcessorEvent.transaction, ProcessorEvent.metric, ProcessorEvent.error];
-
-  const indicesMap = {
-    [ProcessorEvent.transaction]: indices['apm_oss.transactionIndices'],
-    [ProcessorEvent.metric]: indices['apm_oss.metricsIndices'],
-    [ProcessorEvent.error]: indices['apm_oss.errorIndices'],
-  };
-
-  return indexNames.map((name) => indicesMap[name as UIProcessorEvent]);
-}

x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts

@@ -18,7 +18,6 @@ export async function hasHistoricalAgentData(setup: Setup) {
       events: [
         ProcessorEvent.error,
         ProcessorEvent.metric,
-        ProcessorEvent.sourcemap,
         ProcessorEvent.transaction,
       ],
     },

x-pack/plugins/apm/server/plugin.ts

@@ -16,7 +16,8 @@ import {
   Plugin,
   PluginInitializerContext,
 } from 'src/core/server';
-import { APMConfig, APMXPackConfig, mergeConfigs } from '.';
+import { APMConfig, APMXPackConfig } from '.';
+import { mergeConfigs } from './index';
 import { APMOSSPluginSetup } from '../../../../src/plugins/apm_oss/server';
 import { HomeServerPluginSetup } from '../../../../src/plugins/home/server';
 import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';