From fc8ed85798d18c63b3cfbc343588c396eede0fcb Mon Sep 17 00:00:00 2001 From: inge4pres Date: Thu, 24 Feb 2022 11:22:44 +0100 Subject: [PATCH 1/3] downsampled search strategy implementation Signed-off-by: inge4pres --- src/plugins/profiling/server/plugin.ts | 12 +++- .../profiling/server/search/strategy.ts | 70 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 src/plugins/profiling/server/search/strategy.ts diff --git a/src/plugins/profiling/server/plugin.ts b/src/plugins/profiling/server/plugin.ts index 16444ce64197ba..1bdd384fe97a3d 100644 --- a/src/plugins/profiling/server/plugin.ts +++ b/src/plugins/profiling/server/plugin.ts @@ -5,12 +5,14 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'kibana/server'; +import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from 'kibana/server'; import type { DataRequestHandlerContext } from '../../data/server'; import { ProfilingPluginSetupDeps, ProfilingPluginStartDeps } from './types'; import { registerRoutes } from './routes'; +import { DownsampledTopNFactory } from './search/strategy'; +import { DOWNSAMPLED_TOPN_STRATEGY } from '../common/types'; export class ProfilingPlugin implements Plugin @@ -21,14 +23,18 @@ export class ProfilingPlugin this.logger = initializerContext.logger.get(); } - public setup(core: CoreSetup, { data }: ProfilingPluginSetupDeps) { + public setup(core: CoreSetup, deps: ProfilingPluginSetupDeps) { this.logger.debug('profiling: Setup'); // TODO we should create a query here using "data". // We should ensure there are profiling data in the expected indices // and return an error otherwise. // This should be done only once at startup and before exposing the routed APIs. const router = core.http.createRouter(); - core.getStartServices().then(() => { + core.getStartServices().then(([_, depsStart]) => { + deps.data.search.registerSearchStrategy( + DOWNSAMPLED_TOPN_STRATEGY, + DownsampledTopNFactory(depsStart.data) + ); registerRoutes(router, this.logger); }); diff --git a/src/plugins/profiling/server/search/strategy.ts b/src/plugins/profiling/server/search/strategy.ts new file mode 100644 index 00000000000000..ce591212902e4c --- /dev/null +++ b/src/plugins/profiling/server/search/strategy.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; +import { IEsSearchRequest, ISearchStrategy, PluginStart } from '../../../data/server'; +import { autoHistogramSumCountOnGroupByField, newProjectTimeQuery } from '../routes/mappings'; +import { downsampledIndex, getSampledTraceEventsIndex } from '../routes/search_flamechart'; +import { DownsampledRequest, DownsampledTopNResponse } from '../../common/types'; + +export const DownsampledTopNFactory = ( + data: PluginStart +): ISearchStrategy => { + const es = data.search.getSearchStrategy(); + return { + search: async (request, options, deps) => { + const { projectID, timeFrom, timeTo, topNItems, searchField } = request.params!; + const filter = newProjectTimeQuery( + projectID.toString(), + timeFrom.toString(), + timeTo.toString() + ); + + // FIXME these 2 constants should be configurable? + const initialExp = 6; + const targetSampleSize = 20000; // minimum number of samples to get statistically sound results + // Calculate the right down-sampled index to query data from + const sampleCountFromInitialExp = async (): Promise => { + return await deps.esClient.asInternalUser + .search({ + index: downsampledIndex + initialExp, + body: { + query: filter, + size: 0, + track_total_hits: true, + }, + }) + .then((resp) => { + return (resp.body.hits?.total as SearchTotalHits).value; + }); + }; + // Create the query for the actual data + const downsampledReq = { + params: { + index: getSampledTraceEventsIndex( + targetSampleSize, + await sampleCountFromInitialExp(), + initialExp + ).name, + body: { + query: filter, + aggs: { + histogram: autoHistogramSumCountOnGroupByField(searchField, topNItems), + }, + }, + }, + } as IEsSearchRequest; + return es.search(downsampledReq, options, deps); + }, + cancel: async (id, options, deps) => { + if (es.cancel) { + await es.cancel(id, options, deps); + } + }, + }; +}; From ac06ea3dded0b40743e0e3e62e5b23c1842c9476 Mon Sep 17 00:00:00 2001 From: inge4pres Date: Thu, 24 Feb 2022 11:25:09 +0100 Subject: [PATCH 2/3] downsampled search strategy usage on client Signed-off-by: inge4pres --- src/plugins/profiling/public/app.tsx | 2 +- src/plugins/profiling/public/plugin.tsx | 21 +++++++++++++---- src/plugins/profiling/public/services.ts | 30 +++++++++++++++++++++++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/plugins/profiling/public/app.tsx b/src/plugins/profiling/public/app.tsx index 7aace0a4cd7f93..62ea6e44e26b63 100644 --- a/src/plugins/profiling/public/app.tsx +++ b/src/plugins/profiling/public/app.tsx @@ -33,7 +33,7 @@ import { Services } from './services'; type Props = Services; -function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props) { +function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopNData }: Props) { const [topn, setTopN] = useState({ samples: [], series: new Map(), diff --git a/src/plugins/profiling/public/plugin.tsx b/src/plugins/profiling/public/plugin.tsx index 2d828ebb612500..a314488e8d860e 100644 --- a/src/plugins/profiling/public/plugin.tsx +++ b/src/plugins/profiling/public/plugin.tsx @@ -6,18 +6,29 @@ * Side Public License, v 1. */ -import { AppMountParameters, CoreSetup, CoreStart, Plugin } from '../../../core/public'; +import { AppMountParameters, CoreSetup, CoreStart, Plugin } from 'kibana/public'; import { getServices } from './services'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public'; -export class ProdfilerPlugin implements Plugin { - public setup(core: CoreSetup) { +export interface ProdfilerPluginStartDeps { + data: DataPublicPluginStart; +} + +export interface ProdfilerPluginSetupDeps { + data: DataPublicPluginSetup; +} + +export class ProdfilerPlugin + implements Plugin +{ + public setup(core: CoreSetup) { // Register an application into the side navigation menu core.application.register({ id: 'prodfiler', title: 'Prodfiler', async mount({ element }: AppMountParameters) { - const [coreStart] = await core.getStartServices(); - const startServices = getServices(coreStart); + const [coreStart, dataPlugin] = await core.getStartServices(); + const startServices = getServices(coreStart, dataPlugin); const { renderApp } = await import('./app'); return renderApp(startServices, element); }, diff --git a/src/plugins/profiling/public/services.ts b/src/plugins/profiling/public/services.ts index 31911d6b6451b0..97c1536d531ddc 100644 --- a/src/plugins/profiling/public/services.ts +++ b/src/plugins/profiling/public/services.ts @@ -8,11 +8,15 @@ import { CoreStart, HttpFetchError, HttpFetchQuery } from 'kibana/public'; import { getRemoteRoutePaths } from '../common'; +import { ProdfilerPluginStartDeps } from './plugin'; +import { DownsampledRequest, DownsampledTopNResponse } from '../common/types'; export interface Services { fetchTopN: (type: string, seconds: string) => Promise; fetchElasticFlamechart: (seconds: string) => Promise; fetchPixiFlamechart: (seconds: string) => Promise; + // FIXME + fetchTopNData?: (searchField: string, seconds: string) => Promise; } function getFetchQuery(seconds: string): HttpFetchQuery { @@ -27,11 +31,35 @@ function getFetchQuery(seconds: string): HttpFetchQuery { } as HttpFetchQuery; } -export function getServices(core: CoreStart): Services { +export function getServices(core: CoreStart, data?: ProdfilerPluginStartDeps): Services { // To use local fixtures instead, use getLocalRoutePaths const paths = getRemoteRoutePaths(); return { + fetchTopNData: (searchField: string, seconds: string): Promise => { + const unixTime = Math.floor(Date.now() / 1000); + return ( + data!.data.search + .search( + { + params: { + projectID: 5, + timeFrom: unixTime - parseInt(seconds, 10), + timeTo: unixTime, + // FIXME remove hard-coded value for topN items length and expose it through the UI + topNItems: 100, + searchField, + }, + }, + { + strategy: 'downsampledTopN', + } + ) + // get the results and prepare the Promise + .toPromise() + ); + }, + fetchTopN: async (type: string, seconds: string) => { try { const query = getFetchQuery(seconds); From 9d3f2419523c4ce403c32d7a098bcb0cb310cd9d Mon Sep 17 00:00:00 2001 From: inge4pres Date: Fri, 4 Mar 2022 19:37:25 +0100 Subject: [PATCH 3/3] [WIP] broken data plugin usage, can't resolve promise Signed-off-by: inge4pres --- src/plugins/profiling/public/app.tsx | 20 +++++- .../public/components/contexts/topn.tsx | 2 +- .../public/components/stacktrace-nav.tsx | 20 ++++-- src/plugins/profiling/public/services.ts | 66 ++++++++++++------- .../profiling/server/search/strategy.ts | 57 ++++++++++------ 5 files changed, 112 insertions(+), 53 deletions(-) diff --git a/src/plugins/profiling/public/app.tsx b/src/plugins/profiling/public/app.tsx index 62ea6e44e26b63..23fe2af1887fba 100644 --- a/src/plugins/profiling/public/app.tsx +++ b/src/plugins/profiling/public/app.tsx @@ -38,6 +38,10 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopN samples: [], series: new Map(), }); + const [topnData, setTopNData] = useState({ + samples: [], + series: new Map(), + }); const [elasticFlamegraph, setElasticFlamegraph] = useState({ leaves: [] }); const [pixiFlamegraph, setPixiFlamegraph] = useState({}); @@ -45,7 +49,7 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopN const tabs = [ { id: 'stacktrace-elastic', - name: 'Stack Traces (Elastic)', + name: 'Stack Traces (API)', content: ( <> @@ -57,6 +61,20 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopN ), }, + { + id: 'stacktrace-elastic-data', + name: 'Stack Traces (Data Plugin)', + content: ( + <> + + + + + + + + ), + }, { id: 'flamegraph-elastic', name: 'FlameGraph (Elastic)', diff --git a/src/plugins/profiling/public/components/contexts/topn.tsx b/src/plugins/profiling/public/components/contexts/topn.tsx index c4f7c398b20c52..5ea936707fbcac 100644 --- a/src/plugins/profiling/public/components/contexts/topn.tsx +++ b/src/plugins/profiling/public/components/contexts/topn.tsx @@ -8,4 +8,4 @@ import { createContext } from 'react'; -export const TopNContext = createContext(); +export const TopNContext = createContext({}); diff --git a/src/plugins/profiling/public/components/stacktrace-nav.tsx b/src/plugins/profiling/public/components/stacktrace-nav.tsx index 611cd43d5d94e6..5f93c17b31a1fa 100644 --- a/src/plugins/profiling/public/components/stacktrace-nav.tsx +++ b/src/plugins/profiling/public/components/stacktrace-nav.tsx @@ -101,13 +101,19 @@ export const StackTraceNavigation = ({ fetchTopN, setTopN }) => { }); console.log(new Date().toISOString(), 'started payload retrieval'); - fetchTopN(topnValue[0].value, dateValue[0].value).then((response) => { - console.log(new Date().toISOString(), 'finished payload retrieval'); - const samples = getTopN(response); - const series = groupSamplesByCategory(samples); - setTopN({ samples, series }); - console.log(new Date().toISOString(), 'updated local state'); - }); + fetchTopN(topnValue[0].value, dateValue[0].value) + .then((response) => { + console.log(new Date().toISOString(), 'finished payload retrieval'); + const samples = getTopN(response); + const series = groupSamplesByCategory(samples); + console.log('sample %o', samples); + console.log('series %o', series); + setTopN({ samples, series }); + console.log(new Date().toISOString(), 'updated local state'); + }) + .catch((err) => { + console.log('error when reading topN data: ' + err.message); + }); }, [toggleTopNSelected, toggleDateSelected]); return ( diff --git a/src/plugins/profiling/public/services.ts b/src/plugins/profiling/public/services.ts index 97c1536d531ddc..a24f56766311c0 100644 --- a/src/plugins/profiling/public/services.ts +++ b/src/plugins/profiling/public/services.ts @@ -9,14 +9,18 @@ import { CoreStart, HttpFetchError, HttpFetchQuery } from 'kibana/public'; import { getRemoteRoutePaths } from '../common'; import { ProdfilerPluginStartDeps } from './plugin'; -import { DownsampledRequest, DownsampledTopNResponse } from '../common/types'; +import { + DOWNSAMPLED_TOPN_STRATEGY, + DownsampledRequest, + DownsampledTopNResponse, + TopNAggregateResponse, +} from '../common/types'; export interface Services { fetchTopN: (type: string, seconds: string) => Promise; fetchElasticFlamechart: (seconds: string) => Promise; fetchPixiFlamechart: (seconds: string) => Promise; - // FIXME - fetchTopNData?: (searchField: string, seconds: string) => Promise; + fetchTopNData: (searchField: string, seconds: string) => Promise; } function getFetchQuery(seconds: string): HttpFetchQuery { @@ -36,28 +40,44 @@ export function getServices(core: CoreStart, data?: ProdfilerPluginStartDeps): S const paths = getRemoteRoutePaths(); return { - fetchTopNData: (searchField: string, seconds: string): Promise => { + fetchTopNData: async (searchField: string, seconds: string): Promise => { const unixTime = Math.floor(Date.now() / 1000); - return ( - data!.data.search - .search( - { - params: { - projectID: 5, - timeFrom: unixTime - parseInt(seconds, 10), - timeTo: unixTime, - // FIXME remove hard-coded value for topN items length and expose it through the UI - topNItems: 100, - searchField, - }, + const response: TopNAggregateResponse = { topN: { histogram: { buckets: [] } } }; + data!.data.search + .search( + { + params: { + projectID: 5, + timeFrom: unixTime - parseInt(seconds, 10), + timeTo: unixTime, + // FIXME remove hard-coded value for topN items length and expose it through the UI + topNItems: 100, + searchField, }, - { - strategy: 'downsampledTopN', - } - ) - // get the results and prepare the Promise - .toPromise() - ); + }, + { + strategy: DOWNSAMPLED_TOPN_STRATEGY, + } + ) + .subscribe({ + next: (result) => { + console.log('subscription data plugin rawResponse: %o', result.rawResponse); + response.topN.histogram = result.rawResponse.aggregations.histogram; + }, + // TODO error handling + error: (err) => { + console.log('subscription error: %o', err); + }, + // FIXME remove this, used for debugging only + complete: () => { + console.log('subscription completed'); + }, + }); + + console.log('returning Promise of TopNAggregateResponse'); + return await new Promise((resolve, _) => { + return resolve(response); + }); }, fetchTopN: async (type: string, seconds: string) => { diff --git a/src/plugins/profiling/server/search/strategy.ts b/src/plugins/profiling/server/search/strategy.ts index ce591212902e4c..267f4d480ba8e8 100644 --- a/src/plugins/profiling/server/search/strategy.ts +++ b/src/plugins/profiling/server/search/strategy.ts @@ -8,7 +8,11 @@ import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; import { IEsSearchRequest, ISearchStrategy, PluginStart } from '../../../data/server'; -import { autoHistogramSumCountOnGroupByField, newProjectTimeQuery } from '../routes/mappings'; +import { + autoHistogramSumCountOnGroupByField, + newProjectTimeQuery, + ProjectTimeQuery, +} from '../routes/mappings'; import { downsampledIndex, getSampledTraceEventsIndex } from '../routes/search_flamechart'; import { DownsampledRequest, DownsampledTopNResponse } from '../../common/types'; @@ -16,8 +20,37 @@ export const DownsampledTopNFactory = ( data: PluginStart ): ISearchStrategy => { const es = data.search.getSearchStrategy(); + + // FIXME these 2 constants should be configurable? + const initialExp = 6; + const targetSampleSize = 20000; // minimum number of samples to get statistically sound results + + // Calculate the right down-sampled index to query data from + const sampleCountFromInitialExp = (filter: ProjectTimeQuery, options, deps): number => { + // By default, we return no samples and use the un-sampled index + let sampleCount = 0; + es.search( + { + params: { + index: downsampledIndex + initialExp, + body: { + query: filter, + size: 0, + track_total_hits: true, + }, + }, + }, + options, + deps + ).subscribe({ + next: (value) => { + sampleCount = (value.rawResponse.hits.total as SearchTotalHits).value; + }, + }); + return sampleCount; + }; return { - search: async (request, options, deps) => { + search: (request, options, deps) => { const { projectID, timeFrom, timeTo, topNItems, searchField } = request.params!; const filter = newProjectTimeQuery( projectID.toString(), @@ -25,30 +58,12 @@ export const DownsampledTopNFactory = ( timeTo.toString() ); - // FIXME these 2 constants should be configurable? - const initialExp = 6; - const targetSampleSize = 20000; // minimum number of samples to get statistically sound results - // Calculate the right down-sampled index to query data from - const sampleCountFromInitialExp = async (): Promise => { - return await deps.esClient.asInternalUser - .search({ - index: downsampledIndex + initialExp, - body: { - query: filter, - size: 0, - track_total_hits: true, - }, - }) - .then((resp) => { - return (resp.body.hits?.total as SearchTotalHits).value; - }); - }; // Create the query for the actual data const downsampledReq = { params: { index: getSampledTraceEventsIndex( targetSampleSize, - await sampleCountFromInitialExp(), + sampleCountFromInitialExp(filter, options, deps), initialExp ).name, body: {