From 68b5625e5a59131209199704311252f72a16384b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 20 Nov 2020 08:50:34 +0100 Subject: [PATCH] [APM] Correlations UI POC (#82256) --- .../app/Correlations/ErrorCorrelations.tsx | 152 ++++++++++ .../app/Correlations/LatencyCorrelations.tsx | 273 ++++++++++++++++++ .../Correlations/SignificantTermsTable.tsx | 119 ++++++++ .../components/app/Correlations/index.tsx | 127 ++++---- .../app/TransactionDetails/index.tsx | 2 +- .../app/TransactionOverview/index.tsx | 3 +- .../app/service_inventory/index.tsx | 2 +- .../components/shared/Links/url_helpers.ts | 43 +++ x-pack/plugins/apm/readme.md | 6 + .../create_anomaly_detection_jobs.ts | 4 - .../index.ts | 180 ++++++++++++ .../format_top_significant_terms.ts | 44 +++ .../get_charts_for_top_sig_terms.ts | 165 +++++++++++ .../get_duration_for_percentile.ts | 0 .../get_max_latency.ts | 53 ++++ .../index.ts} | 35 +-- .../lib/helpers/get_bucket_size/index.ts | 14 +- .../plugins/apm/server/lib/helpers/metrics.ts | 2 +- .../lib/helpers/transaction_error_rate.ts | 2 + .../java/gc/fetch_and_transform_gc_metrics.ts | 2 +- .../get_service_error_groups/index.ts | 2 +- .../get_services/get_services_items_stats.ts | 3 +- .../get_correlations_for_ranges.ts | 90 ------ .../correlations/get_significant_terms_agg.ts | 68 ----- .../correlations/scoring_rt.ts | 16 - .../lib/transaction_groups/get_error_rate.ts | 2 +- .../charts/get_anomaly_data/index.ts | 2 +- .../charts/get_timeseries_data/fetcher.ts | 2 +- .../charts/get_timeseries_data/index.ts | 2 +- .../plugins/apm/server/routes/correlations.ts | 29 +- .../apm/server/routes/create_apm_api.ts | 4 +- .../basic/tests/correlations/ranges.ts | 95 ------ .../tests/correlations/slow_durations.ts | 115 -------- .../tests/correlations/slow_transactions.ts | 101 +++++++ .../apm_api_integration/basic/tests/index.ts | 3 +- .../tests/service_overview/error_groups.ts | 5 +- .../typings/elasticsearch/aggregations.d.ts | 1 + 37 files changed, 1251 insertions(+), 517 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/Correlations/ErrorCorrelations.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Correlations/LatencyCorrelations.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Correlations/SignificantTermsTable.tsx create mode 100644 x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts create mode 100644 x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/format_top_significant_terms.ts create mode 100644 x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_charts_for_top_sig_terms.ts rename x-pack/plugins/apm/server/lib/{transaction_groups/correlations => correlations/get_correlations_for_slow_transactions}/get_duration_for_percentile.ts (100%) create mode 100644 x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts rename x-pack/plugins/apm/server/lib/{transaction_groups/correlations/get_correlations_for_slow_transactions.ts => correlations/get_correlations_for_slow_transactions/index.ts} (72%) delete mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts delete mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts delete mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts delete mode 100644 x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts delete mode 100644 x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts create mode 100644 x-pack/test/apm_api_integration/basic/tests/correlations/slow_transactions.ts diff --git a/x-pack/plugins/apm/public/components/app/Correlations/ErrorCorrelations.tsx b/x-pack/plugins/apm/public/components/app/Correlations/ErrorCorrelations.tsx new file mode 100644 index 0000000000000..3ad71b52b6037 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Correlations/ErrorCorrelations.tsx @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ScaleType, + Chart, + LineSeries, + Axis, + CurveType, + Position, + timeFormatter, + Settings, +} from '@elastic/charts'; +import React, { useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { EuiTitle, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/useFetcher'; +import { + APIReturnType, + callApmApi, +} from '../../../services/rest/createCallApmApi'; +import { px } from '../../../style/variables'; +import { SignificantTermsTable } from './SignificantTermsTable'; +import { ChartContainer } from '../../shared/charts/chart_container'; + +type CorrelationsApiResponse = NonNullable< + APIReturnType<'GET /api/apm/correlations/failed_transactions'> +>; + +type SignificantTerm = NonNullable< + CorrelationsApiResponse['significantTerms'] +>[0]; + +export function ErrorCorrelations() { + const [ + selectedSignificantTerm, + setSelectedSignificantTerm, + ] = useState(null); + + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { transactionName, transactionType, start, end } = urlParams; + + const { data, status } = useFetcher(() => { + if (start && end) { + return callApmApi({ + endpoint: 'GET /api/apm/correlations/failed_transactions', + params: { + query: { + serviceName, + transactionName, + transactionType, + start, + end, + uiFilters: JSON.stringify(uiFilters), + fieldNames: + 'transaction.name,user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,kubernetes.pod.name,url.domain,container.id,service.node.name', + }, + }, + }); + } + }, [serviceName, start, end, transactionName, transactionType, uiFilters]); + + return ( + <> + + + +

Error rate over time

+ + + + + + + + + ); +} + +function ErrorTimeseriesChart({ + data, + selectedSignificantTerm, + status, +}: { + data?: CorrelationsApiResponse; + selectedSignificantTerm: SignificantTerm | null; + status: FETCH_STATUS; +}) { + const dateFormatter = timeFormatter('HH:mm:ss'); + + return ( + + + + + + `${roundFloat(d * 100)}%`} + /> + + + + {selectedSignificantTerm !== null ? ( + + ) : null} + + + ); +} + +function roundFloat(n: number, digits = 2) { + const factor = Math.pow(10, digits); + return Math.round(n * factor) / factor; +} diff --git a/x-pack/plugins/apm/public/components/app/Correlations/LatencyCorrelations.tsx b/x-pack/plugins/apm/public/components/app/Correlations/LatencyCorrelations.tsx new file mode 100644 index 0000000000000..4364731501b89 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Correlations/LatencyCorrelations.tsx @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ScaleType, + Chart, + LineSeries, + Axis, + CurveType, + BarSeries, + Position, + timeFormatter, + Settings, +} from '@elastic/charts'; +import React, { useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { EuiTitle, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { getDurationFormatter } from '../../../../common/utils/formatters'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/useFetcher'; +import { + APIReturnType, + callApmApi, +} from '../../../services/rest/createCallApmApi'; +import { SignificantTermsTable } from './SignificantTermsTable'; +import { ChartContainer } from '../../shared/charts/chart_container'; + +type CorrelationsApiResponse = NonNullable< + APIReturnType<'GET /api/apm/correlations/slow_transactions'> +>; + +type SignificantTerm = NonNullable< + CorrelationsApiResponse['significantTerms'] +>[0]; + +export function LatencyCorrelations() { + const [ + selectedSignificantTerm, + setSelectedSignificantTerm, + ] = useState(null); + + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { transactionName, transactionType, start, end } = urlParams; + + const { data, status } = useFetcher(() => { + if (start && end) { + return callApmApi({ + endpoint: 'GET /api/apm/correlations/slow_transactions', + params: { + query: { + serviceName, + transactionName, + transactionType, + start, + end, + uiFilters: JSON.stringify(uiFilters), + durationPercentile: '50', + fieldNames: + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,kubernetes.pod.name,url.domain,container.id,service.node.name', + }, + }, + }); + } + }, [serviceName, start, end, transactionName, transactionType, uiFilters]); + + return ( + <> + + + + + +

Average latency over time

+ + + + + +

Latency distribution

+ + + + + + + + + + + ); +} + +function getTimeseriesYMax(data?: CorrelationsApiResponse) { + if (!data?.overall) { + return 0; + } + + const yValues = [ + ...data.overall.timeseries.map((p) => p.y ?? 0), + ...data.significantTerms.flatMap((term) => + term.timeseries.map((p) => p.y ?? 0) + ), + ]; + return Math.max(...yValues); +} + +function getDistributionYMax(data?: CorrelationsApiResponse) { + if (!data?.overall) { + return 0; + } + + const yValues = [ + ...data.overall.distribution.map((p) => p.y ?? 0), + ...data.significantTerms.flatMap((term) => + term.distribution.map((p) => p.y ?? 0) + ), + ]; + return Math.max(...yValues); +} + +function LatencyTimeseriesChart({ + data, + selectedSignificantTerm, + status, +}: { + data?: CorrelationsApiResponse; + selectedSignificantTerm: SignificantTerm | null; + status: FETCH_STATUS; +}) { + const dateFormatter = timeFormatter('HH:mm:ss'); + + const yMax = getTimeseriesYMax(data); + const durationFormatter = getDurationFormatter(yMax); + + return ( + + + + + + durationFormatter(d).formatted} + /> + + + + {selectedSignificantTerm !== null ? ( + + ) : null} + + + ); +} + +function LatencyDistributionChart({ + data, + selectedSignificantTerm, + status, +}: { + data?: CorrelationsApiResponse; + selectedSignificantTerm: SignificantTerm | null; + status: FETCH_STATUS; +}) { + const xMax = Math.max( + ...(data?.overall?.distribution.map((p) => p.x ?? 0) ?? []) + ); + const durationFormatter = getDurationFormatter(xMax); + const yMax = getDistributionYMax(data); + + return ( + + + { + const start = durationFormatter(obj.value); + const end = durationFormatter( + obj.value + data?.distributionInterval + ); + + return `${start.value} - ${end.formatted}`; + }, + }} + /> + durationFormatter(d).formatted} + /> + `${d}%`} + domain={{ min: 0, max: yMax }} + /> + + `${roundFloat(d)}%`} + /> + + {selectedSignificantTerm !== null ? ( + `${roundFloat(d)}%`} + /> + ) : null} + + + ); +} + +function roundFloat(n: number, digits = 2) { + const factor = Math.pow(10, digits); + return Math.round(n * factor) / factor; +} diff --git a/x-pack/plugins/apm/public/components/app/Correlations/SignificantTermsTable.tsx b/x-pack/plugins/apm/public/components/app/Correlations/SignificantTermsTable.tsx new file mode 100644 index 0000000000000..b74517902f89b --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Correlations/SignificantTermsTable.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiBadge, EuiIcon, EuiToolTip, EuiLink } from '@elastic/eui'; +import { useHistory } from 'react-router-dom'; +import { EuiBasicTable } from '@elastic/eui'; +import { asPercent, asInteger } from '../../../../common/utils/formatters'; +import { APIReturnType } from '../../../services/rest/createCallApmApi'; +import { FETCH_STATUS } from '../../../hooks/useFetcher'; +import { createHref } from '../../shared/Links/url_helpers'; + +type CorrelationsApiResponse = + | APIReturnType<'GET /api/apm/correlations/failed_transactions'> + | APIReturnType<'GET /api/apm/correlations/slow_transactions'>; + +type SignificantTerm = NonNullable< + NonNullable['significantTerms'] +>[0]; + +interface Props { + significantTerms?: T[]; + status: FETCH_STATUS; + setSelectedSignificantTerm: (term: T | null) => void; +} + +export function SignificantTermsTable({ + significantTerms, + status, + setSelectedSignificantTerm, +}: Props) { + const history = useHistory(); + const columns = [ + { + field: 'matches', + name: 'Matches', + render: (_: any, term: T) => { + return ( + + <> + 0.03 ? 'primary' : 'secondary' + } + > + {asPercent(term.fgCount, term.bgCount)} + + ({Math.round(term.score)}) + + + ); + }, + }, + { + field: 'fieldName', + name: 'Field name', + }, + { + field: 'filedValue', + name: 'Field value', + render: (_: any, term: T) => String(term.fieldValue).slice(0, 50), + }, + { + field: 'filedValue', + name: '', + render: (_: any, term: T) => { + return ( + <> + + + + + + + + ); + }, + }, + ]; + + return ( + { + return { + onMouseEnter: () => setSelectedSignificantTerm(term), + onMouseLeave: () => setSelectedSignificantTerm(null), + }; + }} + /> + ); +} diff --git a/x-pack/plugins/apm/public/components/app/Correlations/index.tsx b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx index e3dea70a232eb..b0f6b83485e39 100644 --- a/x-pack/plugins/apm/public/components/app/Correlations/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx @@ -4,82 +4,75 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; -import url from 'url'; -import { useParams } from 'react-router-dom'; -import { useLocation } from 'react-router-dom'; -import { EuiTitle, EuiListGroup } from '@elastic/eui'; -import { useUrlParams } from '../../../hooks/useUrlParams'; +import React, { useState } from 'react'; +import { + EuiButtonEmpty, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiTitle, + EuiPortal, + EuiCode, + EuiLink, + EuiCallOut, + EuiButton, +} from '@elastic/eui'; +import { useHistory } from 'react-router-dom'; +import { enableCorrelations } from '../../../../common/ui_settings_keys'; import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; - -const SESSION_STORAGE_KEY = 'apm.debug.show_correlations'; +import { LatencyCorrelations } from './LatencyCorrelations'; +import { ErrorCorrelations } from './ErrorCorrelations'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { createHref } from '../../shared/Links/url_helpers'; export function Correlations() { - const location = useLocation(); - const { serviceName } = useParams<{ serviceName?: string }>(); - const { urlParams, uiFilters } = useUrlParams(); - const { core } = useApmPluginContext(); - const { transactionName, transactionType, start, end } = urlParams; - - if ( - !location.search.includes('&_show_correlations') && - sessionStorage.getItem(SESSION_STORAGE_KEY) !== 'true' - ) { + const { uiSettings } = useApmPluginContext().core; + const { urlParams } = useUrlParams(); + const history = useHistory(); + const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); + if (!uiSettings.get(enableCorrelations)) { return null; } - sessionStorage.setItem(SESSION_STORAGE_KEY, 'true'); - - const query = { - serviceName, - transactionName, - transactionType, - start, - end, - uiFilters: JSON.stringify(uiFilters), - fieldNames: - 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name', - }; - - const listItems = [ - { - label: 'Show correlations between two ranges', - href: url.format({ - query: { - ...query, - gap: 24, - }, - pathname: core.http.basePath.prepend(`/api/apm/correlations/ranges`), - }), - isDisabled: false, - iconType: 'tokenRange', - size: 's' as const, - }, - - { - label: 'Show correlations for slow transactions', - href: url.format({ - query: { - ...query, - durationPercentile: 95, - }, - pathname: core.http.basePath.prepend( - `/api/apm/correlations/slow_durations` - ), - }), - isDisabled: false, - iconType: 'clock', - size: 's' as const, - }, - ]; - return ( <> - -

Correlations

- + { + setIsFlyoutVisible(true); + }} + > + View correlations + + + {isFlyoutVisible && ( + + setIsFlyoutVisible(false)} + > + + +

Correlations

+ + + + {urlParams.kuery ? ( + + Filtering by + {urlParams.kuery} + + Clear + + + ) : null} - + + + + + + )} ); } diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx index 9d9261fec6c1e..cc6bacc4f3ccb 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx @@ -119,9 +119,9 @@ export function TransactionDetails({ - + diff --git a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx index a4f8d37867dd5..8208916c20337 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx @@ -123,10 +123,11 @@ export function TransactionOverview({ serviceName }: TransactionOverviewProps) { return ( <> - + + - + diff --git a/x-pack/plugins/apm/public/components/shared/Links/url_helpers.ts b/x-pack/plugins/apm/public/components/shared/Links/url_helpers.ts index 991735a450724..9da26b3fcefac 100644 --- a/x-pack/plugins/apm/public/components/shared/Links/url_helpers.ts +++ b/x-pack/plugins/apm/public/components/shared/Links/url_helpers.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { History } from 'history'; import { parse, stringify } from 'query-string'; import { url } from '../../../../../../../src/plugins/kibana_utils/public'; import { LocalUIFilterName } from '../../../../common/ui_filter'; @@ -20,6 +21,48 @@ export function fromQuery(query: Record) { return stringify(encodedQuery, { sort: false, encode: false }); } +type LocationWithQuery = Partial< + History['location'] & { + query: Record; + } +>; + +function getNextLocation( + history: History, + locationWithQuery: LocationWithQuery +) { + const { query, ...rest } = locationWithQuery; + return { + ...history.location, + ...rest, + search: fromQuery({ + ...toQuery(history.location.search), + ...query, + }), + }; +} + +export function replace( + history: History, + locationWithQuery: LocationWithQuery +) { + const location = getNextLocation(history, locationWithQuery); + return history.replace(location); +} + +export function push(history: History, locationWithQuery: LocationWithQuery) { + const location = getNextLocation(history, locationWithQuery); + return history.push(location); +} + +export function createHref( + history: History, + locationWithQuery: LocationWithQuery +) { + const location = getNextLocation(history, locationWithQuery); + return history.createHref(location); +} + export type APMQueryParams = { transactionId?: string; transactionName?: string; diff --git a/x-pack/plugins/apm/readme.md b/x-pack/plugins/apm/readme.md index 0adfb99e7164e..00d7e8e1dd5e4 100644 --- a/x-pack/plugins/apm/readme.md +++ b/x-pack/plugins/apm/readme.md @@ -115,6 +115,12 @@ For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme) _Note: Run the following commands from `kibana/`._ +### Typescript + +``` +yarn tsc --noEmit --project x-pack/tsconfig.json --skipLibCheck +``` + ### Prettier ``` diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts index a10762622b2c6..449aa88752f21 100644 --- a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts @@ -10,7 +10,6 @@ import { snakeCase } from 'lodash'; import Boom from '@hapi/boom'; import { ProcessorEvent } from '../../../common/processor_event'; import { ML_ERRORS } from '../../../common/anomaly_detection'; -import { PromiseReturnType } from '../../../../observability/typings/common'; import { Setup } from '../helpers/setup_request'; import { TRANSACTION_DURATION, @@ -19,9 +18,6 @@ import { import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants'; import { getEnvironmentUiFilterES } from '../helpers/convert_ui_filters/get_environment_ui_filter_es'; -export type CreateAnomalyDetectionJobsAPIResponse = PromiseReturnType< - typeof createAnomalyDetectionJobs ->; export async function createAnomalyDetectionJobs( setup: Setup, environments: string[], diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts new file mode 100644 index 0000000000000..ba739310bc342 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEmpty } from 'lodash'; +import { EventOutcome } from '../../../../common/event_outcome'; +import { + formatTopSignificantTerms, + TopSigTerm, +} from '../get_correlations_for_slow_transactions/format_top_significant_terms'; +import { AggregationOptionsByType } from '../../../../../../typings/elasticsearch/aggregations'; +import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { + EVENT_OUTCOME, + SERVICE_NAME, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { getBucketSize } from '../../helpers/get_bucket_size'; +import { + getOutcomeAggregation, + getTransactionErrorRateTimeSeries, +} from '../../helpers/transaction_error_rate'; + +export async function getCorrelationsForFailedTransactions({ + serviceName, + transactionType, + transactionName, + fieldNames, + setup, +}: { + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; + fieldNames: string[]; + setup: Setup & SetupTimeRange; +}) { + const { start, end, esFilter, apmEventClient } = setup; + + const backgroundFilters: ESFilter[] = [ + ...esFilter, + { range: rangeFilter(start, end) }, + ]; + + if (serviceName) { + backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + // foreground filters + filter: [ + ...backgroundFilters, + { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, + ], + }, + }, + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { bool: { filter: backgroundFilters } }, + }, + }, + }; + }, {} as Record), + }, + }; + + const response = await apmEventClient.search(params); + const topSigTerms = formatTopSignificantTerms(response.aggregations); + return getChartsForTopSigTerms({ setup, backgroundFilters, topSigTerms }); +} + +export async function getChartsForTopSigTerms({ + setup, + backgroundFilters, + topSigTerms, +}: { + setup: Setup & SetupTimeRange; + backgroundFilters: ESFilter[]; + topSigTerms: TopSigTerm[]; +}) { + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 30 }); + + if (isEmpty(topSigTerms)) { + return {}; + } + + const timeseriesAgg = { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: { + // TODO: add support for metrics + outcomes: getOutcomeAggregation({ searchAggregatedTransactions: false }), + }, + }; + + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { timeseries: timeseriesAgg }, + }; + return acc; + }, + {} as Record< + string, + { + filter: AggregationOptionsByType['filter']; + aggs: { timeseries: typeof timeseriesAgg }; + } + > + ); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: backgroundFilters } }, + aggs: { + // overall aggs + timeseries: timeseriesAgg, + + // per term aggs + ...perTermAggs, + }, + }, + }; + + const response = await apmEventClient.search(params); + type Agg = NonNullable; + + if (!response.aggregations) { + return {}; + } + + return { + overall: { + timeseries: getTransactionErrorRateTimeSeries( + response.aggregations.timeseries.buckets + ), + }, + significantTerms: topSigTerms.map((topSig, index) => { + // @ts-expect-error + const agg = response.aggregations[`term_${index}`] as Agg; + + return { + ...topSig, + timeseries: getTransactionErrorRateTimeSeries(agg.timeseries.buckets), + }; + }), + }; +} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/format_top_significant_terms.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/format_top_significant_terms.ts new file mode 100644 index 0000000000000..f168b49fb18fd --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/format_top_significant_terms.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { orderBy } from 'lodash'; +import { + AggregationOptionsByType, + AggregationResultOf, +} from '../../../../../../typings/elasticsearch/aggregations'; + +export interface TopSigTerm { + bgCount: number; + fgCount: number; + fieldName: string; + fieldValue: string | number; + score: number; +} + +type SigTermAggs = AggregationResultOf< + { significant_terms: AggregationOptionsByType['significant_terms'] }, + {} +>; + +export function formatTopSignificantTerms( + aggregations?: Record +) { + const significantTerms = Object.entries(aggregations ?? []).flatMap( + ([fieldName, agg]) => { + return agg.buckets.map((bucket) => ({ + fieldName, + fieldValue: bucket.key, + bgCount: bucket.bg_count, + fgCount: bucket.doc_count, + score: bucket.score, + })); + } + ); + + // get top 10 terms ordered by score + const topSigTerms = orderBy(significantTerms, 'score', 'desc').slice(0, 10); + return topSigTerms; +} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_charts_for_top_sig_terms.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_charts_for_top_sig_terms.ts new file mode 100644 index 0000000000000..cbefd5e2133e5 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_charts_for_top_sig_terms.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEmpty } from 'lodash'; +import { AggregationOptionsByType } from '../../../../../../typings/elasticsearch/aggregations'; +import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { getBucketSize } from '../../helpers/get_bucket_size'; +import { TopSigTerm } from './format_top_significant_terms'; +import { getMaxLatency } from './get_max_latency'; + +export async function getChartsForTopSigTerms({ + setup, + backgroundFilters, + topSigTerms, +}: { + setup: Setup & SetupTimeRange; + backgroundFilters: ESFilter[]; + topSigTerms: TopSigTerm[]; +}) { + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 30 }); + + if (isEmpty(topSigTerms)) { + return {}; + } + + const maxLatency = await getMaxLatency({ + setup, + backgroundFilters, + topSigTerms, + }); + + if (!maxLatency) { + return {}; + } + + const intervalBuckets = 20; + const distributionInterval = roundtoTenth(maxLatency / intervalBuckets); + + const distributionAgg = { + // filter out outliers not included in the significant term docs + filter: { range: { [TRANSACTION_DURATION]: { lte: maxLatency } } }, + aggs: { + dist_filtered_by_latency: { + histogram: { + // TODO: add support for metrics + field: TRANSACTION_DURATION, + interval: distributionInterval, + min_doc_count: 0, + extended_bounds: { + min: 0, + max: maxLatency, + }, + }, + }, + }, + }; + + const timeseriesAgg = { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: { + average: { + avg: { + // TODO: add support for metrics + field: TRANSACTION_DURATION, + }, + }, + }, + }; + + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { + distribution: distributionAgg, + timeseries: timeseriesAgg, + }, + }; + return acc; + }, + {} as Record< + string, + { + filter: AggregationOptionsByType['filter']; + aggs: { + distribution: typeof distributionAgg; + timeseries: typeof timeseriesAgg; + }; + } + > + ); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: backgroundFilters } }, + aggs: { + // overall aggs + distribution: distributionAgg, + timeseries: timeseriesAgg, + + // per term aggs + ...perTermAggs, + }, + }, + }; + + const response = await apmEventClient.search(params); + type Agg = NonNullable; + + if (!response.aggregations) { + return; + } + + function formatTimeseries(timeseries: Agg['timeseries']) { + return timeseries.buckets.map((bucket) => ({ + x: bucket.key, + y: bucket.average.value, + })); + } + + function formatDistribution(distribution: Agg['distribution']) { + const total = distribution.doc_count; + return distribution.dist_filtered_by_latency.buckets.map((bucket) => ({ + x: bucket.key, + y: (bucket.doc_count / total) * 100, + })); + } + + return { + distributionInterval, + overall: { + timeseries: formatTimeseries(response.aggregations.timeseries), + distribution: formatDistribution(response.aggregations.distribution), + }, + significantTerms: topSigTerms.map((topSig, index) => { + // @ts-expect-error + const agg = response.aggregations[`term_${index}`] as Agg; + + return { + ...topSig, + timeseries: formatTimeseries(agg.timeseries), + distribution: formatDistribution(agg.distribution), + }; + }), + }; +} + +function roundtoTenth(v: number) { + return Math.pow(10, Math.round(Math.log10(v))); +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_duration_for_percentile.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts rename to x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_duration_for_percentile.ts diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts new file mode 100644 index 0000000000000..3f86d2900e85b --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { TopSigTerm } from './format_top_significant_terms'; + +export async function getMaxLatency({ + setup, + backgroundFilters, + topSigTerms, +}: { + setup: Setup & SetupTimeRange; + backgroundFilters: ESFilter[]; + topSigTerms: TopSigTerm[]; +}) { + const { apmEventClient } = setup; + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + filter: backgroundFilters, + + // only include docs containing the significant terms + should: topSigTerms.map((term) => ({ + term: { [term.fieldName]: term.fieldValue }, + })), + minimum_should_match: 1, + }, + }, + aggs: { + // TODO: add support for metrics + // max_latency: { max: { field: TRANSACTION_DURATION } }, + max_latency: { + percentiles: { field: TRANSACTION_DURATION, percents: [99] }, + }, + }, + }, + }; + + const response = await apmEventClient.search(params); + // return response.aggregations?.max_latency.value; + return Object.values(response.aggregations?.max_latency.values ?? {})[0]; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts similarity index 72% rename from x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts rename to x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts index 76e595c928cf2..b8a5ab93591a4 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts @@ -4,7 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +import { AggregationOptionsByType } from '../../../../../../typings/elasticsearch/aggregations'; import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { rangeFilter } from '../../../../common/utils/range_filter'; import { SERVICE_NAME, TRANSACTION_DURATION, @@ -12,15 +14,10 @@ import { TRANSACTION_TYPE, } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { asDuration } from '../../../../common/utils/formatters'; -import { rangeFilter } from '../../../../common/utils/range_filter'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getDurationForPercentile } from './get_duration_for_percentile'; -import { - formatAggregationResponse, - getSignificantTermsAgg, -} from './get_significant_terms_agg'; -import { SignificantTermsScoring } from './scoring_rt'; +import { formatTopSignificantTerms } from './format_top_significant_terms'; +import { getChartsForTopSigTerms } from './get_charts_for_top_sig_terms'; export async function getCorrelationsForSlowTransactions({ serviceName, @@ -28,13 +25,11 @@ export async function getCorrelationsForSlowTransactions({ transactionName, durationPercentile, fieldNames, - scoring, setup, }: { serviceName: string | undefined; transactionType: string | undefined; transactionName: string | undefined; - scoring: SignificantTermsScoring; durationPercentile: number; fieldNames: string[]; setup: Setup & SetupTimeRange; @@ -79,16 +74,22 @@ export async function getCorrelationsForSlowTransactions({ ], }, }, - aggs: getSignificantTermsAgg({ fieldNames, backgroundFilters, scoring }), + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { bool: { filter: backgroundFilters } }, + }, + }, + }; + }, {} as Record), }, }; const response = await apmEventClient.search(params); - - return { - message: `Showing significant fields for transactions slower than ${durationPercentile}th percentile (${asDuration( - durationForPercentile - )})`, - response: formatAggregationResponse(response.aggregations), - }; + const topSigTerms = formatTopSignificantTerms(response.aggregations); + return getChartsForTopSigTerms({ setup, backgroundFilters, topSigTerms }); } diff --git a/x-pack/plugins/apm/server/lib/helpers/get_bucket_size/index.ts b/x-pack/plugins/apm/server/lib/helpers/get_bucket_size/index.ts index 5b78d97d5b681..2a891bc6f8990 100644 --- a/x-pack/plugins/apm/server/lib/helpers/get_bucket_size/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/get_bucket_size/index.ts @@ -8,11 +8,15 @@ import moment from 'moment'; // @ts-expect-error import { calculateAuto } from './calculate_auto'; -export function getBucketSize( - start: number, - end: number, - numBuckets: number = 100 -) { +export function getBucketSize({ + start, + end, + numBuckets = 100, +}: { + start: number; + end: number; + numBuckets?: number; +}) { const duration = moment.duration(end - start, 'ms'); const bucketSize = Math.max( calculateAuto.near(numBuckets, duration).asSeconds(), diff --git a/x-pack/plugins/apm/server/lib/helpers/metrics.ts b/x-pack/plugins/apm/server/lib/helpers/metrics.ts index ea018868f9517..7ea8dc35b41d0 100644 --- a/x-pack/plugins/apm/server/lib/helpers/metrics.ts +++ b/x-pack/plugins/apm/server/lib/helpers/metrics.ts @@ -11,7 +11,7 @@ export function getMetricsDateHistogramParams( end: number, metricsInterval: number ) { - const { bucketSize } = getBucketSize(start, end); + const { bucketSize } = getBucketSize({ start, end }); return { field: '@timestamp', diff --git a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts index 03a44e77ba2d3..536be56e152a3 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts @@ -20,6 +20,8 @@ export function getOutcomeAggregation({ return { terms: { field: EVENT_OUTCOME }, aggs: { + // simply using the doc count to get the number of requests is not possible for transaction metrics (histograms) + // to work around this we get the number of transactions by counting the number of latency values count: { value_count: { field: getTransactionDurationFieldForAggregatedTransactions( diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts index 2ed11480a7585..10aa56e79f06b 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts @@ -40,7 +40,7 @@ export async function fetchAndTransformGcMetrics({ }) { const { start, end, apmEventClient, config } = setup; - const { bucketSize } = getBucketSize(start, end); + const { bucketSize } = getBucketSize({ start, end }); const projection = getMetricsProjection({ setup, diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts index 99d978116180b..0ca085105c30c 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts @@ -43,7 +43,7 @@ export async function getServiceErrorGroups({ }) { const { apmEventClient, start, end, esFilter } = setup; - const { intervalString } = getBucketSize(start, end, numBuckets); + const { intervalString } = getBucketSize({ start, end, numBuckets }); const response = await apmEventClient.search({ apm: { diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts index fac80cf22c310..c8ebaa13d9df9 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts @@ -37,7 +37,8 @@ import { function getDateHistogramOpts(start: number, end: number) { return { field: '@timestamp', - fixed_interval: getBucketSize(start, end, 20).intervalString, + fixed_interval: getBucketSize({ start, end, numBuckets: 20 }) + .intervalString, min_doc_count: 0, extended_bounds: { min: start, max: end }, }; diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts deleted file mode 100644 index 3cf0271baa1c6..0000000000000 --- a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { rangeFilter } from '../../../../common/utils/range_filter'; -import { - SERVICE_NAME, - TRANSACTION_NAME, - TRANSACTION_TYPE, -} from '../../../../common/elasticsearch_fieldnames'; -import { ProcessorEvent } from '../../../../common/processor_event'; -import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { - getSignificantTermsAgg, - formatAggregationResponse, -} from './get_significant_terms_agg'; -import { SignificantTermsScoring } from './scoring_rt'; - -export async function getCorrelationsForRanges({ - serviceName, - transactionType, - transactionName, - scoring, - gapBetweenRanges, - fieldNames, - setup, -}: { - serviceName: string | undefined; - transactionType: string | undefined; - transactionName: string | undefined; - scoring: SignificantTermsScoring; - gapBetweenRanges: number; - fieldNames: string[]; - setup: Setup & SetupTimeRange; -}) { - const { start, end, esFilter, apmEventClient } = setup; - - const baseFilters = [...esFilter]; - - if (serviceName) { - baseFilters.push({ term: { [SERVICE_NAME]: serviceName } }); - } - - if (transactionType) { - baseFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); - } - - if (transactionName) { - baseFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); - } - - const diff = end - start + gapBetweenRanges; - const baseRangeStart = start - diff; - const baseRangeEnd = end - diff; - const backgroundFilters = [ - ...baseFilters, - { range: rangeFilter(baseRangeStart, baseRangeEnd) }, - ]; - - const params = { - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { - bool: { filter: [...baseFilters, { range: rangeFilter(start, end) }] }, - }, - aggs: getSignificantTermsAgg({ - fieldNames, - backgroundFilters, - backgroundIsSuperset: false, - scoring, - }), - }, - }; - - const response = await apmEventClient.search(params); - - return { - message: `Showing significant fields between the ranges`, - firstRange: `${new Date(baseRangeStart).toISOString()} - ${new Date( - baseRangeEnd - ).toISOString()}`, - lastRange: `${new Date(start).toISOString()} - ${new Date( - end - ).toISOString()}`, - response: formatAggregationResponse(response.aggregations), - }; -} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts deleted file mode 100644 index c5ab8d8f1d111..0000000000000 --- a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ESFilter } from '../../../../../../typings/elasticsearch'; -import { SignificantTermsScoring } from './scoring_rt'; - -export function getSignificantTermsAgg({ - fieldNames, - backgroundFilters, - backgroundIsSuperset = true, - scoring = 'percentage', -}: { - fieldNames: string[]; - backgroundFilters: ESFilter[]; - backgroundIsSuperset?: boolean; - scoring: SignificantTermsScoring; -}) { - return fieldNames.reduce((acc, fieldName) => { - return { - ...acc, - [fieldName]: { - significant_terms: { - size: 10, - field: fieldName, - background_filter: { bool: { filter: backgroundFilters } }, - - // indicate whether background is a superset of the foreground - mutual_information: { background_is_superset: backgroundIsSuperset }, - - // different scorings https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-significantterms-aggregation.html#significantterms-aggregation-parameters - [scoring]: {}, - min_doc_count: 5, - shard_min_doc_count: 5, - }, - }, - [`cardinality-${fieldName}`]: { - cardinality: { field: fieldName }, - }, - }; - }, {} as Record); -} - -export function formatAggregationResponse(aggs?: Record) { - if (!aggs) { - return; - } - - return Object.entries(aggs).reduce((acc, [key, value]) => { - if (key.startsWith('cardinality-')) { - if (value.value > 0) { - const fieldName = key.slice(12); - acc[fieldName] = { - ...acc[fieldName], - cardinality: value.value, - }; - } - } else if (value.buckets.length > 0) { - acc[key] = { - ...acc[key], - value, - }; - } - return acc; - }, {} as Record); -} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts deleted file mode 100644 index cb94b6251eb07..0000000000000 --- a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import * as t from 'io-ts'; - -export const scoringRt = t.union([ - t.literal('jlh'), - t.literal('chi_square'), - t.literal('gnd'), - t.literal('percentage'), -]); - -export type SignificantTermsScoring = t.TypeOf; diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts index e9d273dad6262..dfd11203b87f1 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts @@ -78,7 +78,7 @@ export async function getErrorRate({ timeseries: { date_histogram: { field: '@timestamp', - fixed_interval: getBucketSize(start, end).intervalString, + fixed_interval: getBucketSize({ start, end }).intervalString, min_doc_count: 0, extended_bounds: { min: start, max: end }, }, diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts index f11623eaa2dae..e72219a3cbd72 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts @@ -77,7 +77,7 @@ export async function getAnomalySeries({ return; } - const { intervalString, bucketSize } = getBucketSize(start, end); + const { intervalString, bucketSize } = getBucketSize({ start, end }); const esResponse = await anomalySeriesFetcher({ serviceName, diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.ts index a2da3977b81c7..cffec688806b5 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.ts @@ -36,7 +36,7 @@ export function timeseriesFetcher({ searchAggregatedTransactions: boolean; }) { const { start, end, apmEventClient } = setup; - const { intervalString } = getBucketSize(start, end); + const { intervalString } = getBucketSize({ start, end }); const filter: ESFilter[] = [ { term: { [SERVICE_NAME]: serviceName } }, diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/index.ts index c0421005dd06e..6c923290848a1 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/index.ts @@ -17,7 +17,7 @@ export async function getApmTimeseriesData(options: { searchAggregatedTransactions: boolean; }) { const { start, end } = options.setup; - const { bucketSize } = getBucketSize(start, end); + const { bucketSize } = getBucketSize({ start, end }); const durationAsMinutes = (end - start) / 1000 / 60; const timeseriesResponse = await timeseriesFetcher(options); diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 99fb615d310db..6d1aead9292e3 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -6,21 +6,19 @@ import * as t from 'io-ts'; import { rangeRt } from './default_api_types'; -import { getCorrelationsForSlowTransactions } from '../lib/transaction_groups/correlations/get_correlations_for_slow_transactions'; -import { getCorrelationsForRanges } from '../lib/transaction_groups/correlations/get_correlations_for_ranges'; -import { scoringRt } from '../lib/transaction_groups/correlations/scoring_rt'; +import { getCorrelationsForSlowTransactions } from '../lib/correlations/get_correlations_for_slow_transactions'; +import { getCorrelationsForFailedTransactions } from '../lib/correlations/get_correlations_for_failed_transactions'; import { createRoute } from './create_route'; import { setupRequest } from '../lib/helpers/setup_request'; export const correlationsForSlowTransactionsRoute = createRoute({ - endpoint: 'GET /api/apm/correlations/slow_durations', + endpoint: 'GET /api/apm/correlations/slow_transactions', params: t.type({ query: t.intersection([ t.partial({ serviceName: t.string, transactionName: t.string, transactionType: t.string, - scoring: scoringRt, }), t.type({ durationPercentile: t.string, @@ -39,7 +37,6 @@ export const correlationsForSlowTransactionsRoute = createRoute({ transactionName, durationPercentile, fieldNames, - scoring = 'percentage', } = context.params.query; return getCorrelationsForSlowTransactions({ @@ -48,22 +45,19 @@ export const correlationsForSlowTransactionsRoute = createRoute({ transactionName, durationPercentile: parseInt(durationPercentile, 10), fieldNames: fieldNames.split(','), - scoring, setup, }); }, }); -export const correlationsForRangesRoute = createRoute({ - endpoint: 'GET /api/apm/correlations/ranges', +export const correlationsForFailedTransactionsRoute = createRoute({ + endpoint: 'GET /api/apm/correlations/failed_transactions', params: t.type({ query: t.intersection([ t.partial({ serviceName: t.string, transactionName: t.string, transactionType: t.string, - scoring: scoringRt, - gap: t.string, }), t.type({ fieldNames: t.string, @@ -75,27 +69,18 @@ export const correlationsForRangesRoute = createRoute({ options: { tags: ['access:apm'] }, handler: async ({ context, request }) => { const setup = await setupRequest(context, request); - const { serviceName, transactionType, transactionName, - scoring = 'percentage', - gap, + fieldNames, } = context.params.query; - const gapBetweenRanges = parseInt(gap || '0', 10) * 3600 * 1000; - if (gapBetweenRanges < 0) { - throw new Error('gap must be 0 or positive'); - } - - return getCorrelationsForRanges({ + return getCorrelationsForFailedTransactions({ serviceName, transactionType, transactionName, - scoring, - gapBetweenRanges, fieldNames: fieldNames.split(','), setup, }); diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index a272b448deaf1..1edbae474d1c8 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -43,8 +43,8 @@ import { serviceNodesRoute } from './service_nodes'; import { tracesRoute, tracesByIdRoute } from './traces'; import { transactionByTraceIdRoute } from './transaction'; import { - correlationsForRangesRoute, correlationsForSlowTransactionsRoute, + correlationsForFailedTransactionsRoute, } from './correlations'; import { transactionGroupsBreakdownRoute, @@ -129,7 +129,7 @@ const createApmApi = () => { // Correlations .add(correlationsForSlowTransactionsRoute) - .add(correlationsForRangesRoute) + .add(correlationsForFailedTransactionsRoute) // APM indices .add(apmIndexSettingsRoute) diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts deleted file mode 100644 index 751ee8753c449..0000000000000 --- a/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import { format } from 'url'; -import { PromiseReturnType } from '../../../../../plugins/observability/typings/common'; -import archives_metadata from '../../../common/archives_metadata'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); - const archiveName = 'apm_8.0.0'; - const range = archives_metadata[archiveName]; - - // url parameters - const start = '2020-09-29T14:45:00.000Z'; - const end = range.end; - const fieldNames = - 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; - - describe('Ranges', () => { - const url = format({ - pathname: `/api/apm/correlations/ranges`, - query: { start, end, fieldNames }, - }); - - describe('when data is not loaded ', () => { - it('handles the empty state', async () => { - const response = await supertest.get(url); - expect(response.status).to.be(200); - expect(response.body.response).to.be(undefined); - }); - }); - - describe('when data is loaded', () => { - let response: PromiseReturnType; - before(async () => { - await esArchiver.load(archiveName); - response = await supertest.get(url); - }); - - after(() => esArchiver.unload(archiveName)); - - it('returns successfully', () => { - expect(response.status).to.eql(200); - }); - - it('returns fields in response', () => { - expectSnapshot(Object.keys(response.body.response)).toMatchInline(` - Array [ - "service.node.name", - "host.ip", - "user.id", - "user_agent.name", - "container.id", - "url.domain", - ] - `); - }); - - it('returns cardinality for each field', () => { - const cardinalitys = Object.values(response.body.response).map( - (field: any) => field.cardinality - ); - - expectSnapshot(cardinalitys).toMatchInline(` - Array [ - 5, - 6, - 20, - 6, - 5, - 4, - ] - `); - }); - - it('returns buckets', () => { - const { buckets } = response.body.response['user.id'].value; - expectSnapshot(buckets[0]).toMatchInline(` - Object { - "bg_count": 2, - "doc_count": 7, - "key": "20", - "score": 3.5, - } - `); - }); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts deleted file mode 100644 index 3cf1c2cecb42b..0000000000000 --- a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import { format } from 'url'; -import { PromiseReturnType } from '../../../../../plugins/observability/typings/common'; -import archives_metadata from '../../../common/archives_metadata'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); - const archiveName = 'apm_8.0.0'; - const range = archives_metadata[archiveName]; - - // url parameters - const start = range.start; - const end = range.end; - const durationPercentile = 95; - const fieldNames = - 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; - - // Failing: See https://github.com/elastic/kibana/issues/81264 - describe('Slow durations', () => { - const url = format({ - pathname: `/api/apm/correlations/slow_durations`, - query: { start, end, durationPercentile, fieldNames }, - }); - - describe('when data is not loaded ', () => { - it('handles the empty state', async () => { - const response = await supertest.get(url); - - expect(response.status).to.be(200); - expect(response.body.response).to.be(undefined); - }); - }); - - describe('when data is loaded', () => { - before(() => esArchiver.load(archiveName)); - after(() => esArchiver.unload(archiveName)); - - describe('making request with default args', () => { - let response: PromiseReturnType; - before(async () => { - response = await supertest.get(url); - - it('returns successfully', () => { - expect(response.status).to.eql(200); - }); - - it('returns fields in response', () => { - expectSnapshot(Object.keys(response.body.response)).toMatchInline(` - Array [ - "service.node.name", - "host.ip", - "user.id", - "user_agent.name", - "container.id", - "url.domain", - ] - `); - }); - - it('returns cardinality for each field', () => { - const cardinalitys = Object.values(response.body.response).map( - (field: any) => field.cardinality - ); - - expectSnapshot(cardinalitys).toMatchInline(` - Array [ - 5, - 6, - 3, - 5, - 5, - 4, - ] - `); - }); - - it('returns buckets', () => { - const { buckets } = response.body.response['user.id'].value; - expectSnapshot(buckets[0]).toMatchInline(` - Object { - "bg_count": 32, - "doc_count": 6, - "key": "2", - "score": 0.1875, - } - `); - }); - }); - }); - - describe('making a request for each "scoring"', () => { - ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { - it(`returns response for scoring "${scoring}"`, async () => { - const response = await supertest.get( - format({ - pathname: `/api/apm/correlations/slow_durations`, - query: { start, end, durationPercentile, fieldNames, scoring }, - }) - ); - - expect(response.status).to.be(200); - }); - }); - }); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_transactions.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_transactions.ts new file mode 100644 index 0000000000000..c0978db69a3c9 --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_transactions.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { APIReturnType } from '../../../../../plugins/apm/public/services/rest/createCallApmApi'; +import archives_metadata from '../../../common/archives_metadata'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + describe('Slow durations', () => { + const url = format({ + pathname: `/api/apm/correlations/slow_transactions`, + query: { + start: range.start, + end: range.end, + durationPercentile: 95, + fieldNames: + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name', + }, + }); + + describe('when data is not loaded', () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + }); + + describe('when data is loaded', () => { + before(() => esArchiver.load(archiveName)); + after(() => esArchiver.unload(archiveName)); + + describe('making request with default args', () => { + type ResponseBody = APIReturnType<'GET /api/apm/correlations/slow_transactions'>; + let response: { + status: number; + body: NonNullable; + }; + + before(async () => { + response = await supertest.get(url); + }); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns significant terms', () => { + expectSnapshot(response.body?.significantTerms?.map((term) => term.fieldName)) + .toMatchInline(` + Array [ + "host.ip", + "service.node.name", + "container.id", + "url.domain", + "user_agent.name", + "user.id", + "host.ip", + "service.node.name", + "container.id", + "user.id", + ] + `); + }); + + it('returns a timeseries per term', () => { + // @ts-ignore + expectSnapshot(response.body?.significantTerms[0].timeseries.length).toMatchInline(`31`); + }); + + it('returns a distribution per term', () => { + // @ts-ignore + expectSnapshot(response.body?.significantTerms[0].distribution.length).toMatchInline( + `11` + ); + }); + + it('returns overall timeseries', () => { + // @ts-ignore + expectSnapshot(response.body?.overall.timeseries.length).toMatchInline(`31`); + }); + + it('returns overall distribution', () => { + // @ts-ignore + expectSnapshot(response.body?.overall.distribution.length).toMatchInline(`11`); + }); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index 0381e5f51bb9b..a8e3f2832ec4e 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -59,8 +59,7 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont }); describe('Correlations', function () { - loadTestFile(require.resolve('./correlations/slow_durations')); - loadTestFile(require.resolve('./correlations/ranges')); + loadTestFile(require.resolve('./correlations/slow_transactions')); }); }); } diff --git a/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts index 088b7cb8bb568..6d0d1e4b52bec 100644 --- a/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts +++ b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts @@ -99,9 +99,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { } `); - expectSnapshot( - firstItem.occurrences.timeseries.filter(({ y }: any) => y > 0).length - ).toMatchInline(`7`); + const visibleDataPoints = firstItem.occurrences.timeseries.filter(({ y }: any) => y > 0); + expectSnapshot(visibleDataPoints.length).toMatchInline(`7`); }); it('sorts items in the correct order', async () => { diff --git a/x-pack/typings/elasticsearch/aggregations.d.ts b/x-pack/typings/elasticsearch/aggregations.d.ts index bc9ed447c8717..f471b83fbbc6b 100644 --- a/x-pack/typings/elasticsearch/aggregations.d.ts +++ b/x-pack/typings/elasticsearch/aggregations.d.ts @@ -354,6 +354,7 @@ interface AggregationResponsePart