Use Kibana data plugin and search strategy

src/plugins/profiling/public/app.tsx

@@ -33,19 +33,23 @@ import { Services } from './services';
 
 type Props = Services;
 
-function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props) {
+function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopNData }: Props) {
  const [topn, setTopN] = useState({
  samples: [],
  series: new Map(),
  });
+ const [topnData, setTopNData] = useState({
+ samples: [],
+ series: new Map(),
+ });
 
  const [elasticFlamegraph, setElasticFlamegraph] = useState({ leaves: [] });
  const [pixiFlamegraph, setPixiFlamegraph] = useState({});
 
  const tabs = [
  {
  id: 'stacktrace-elastic',
- name: 'Stack Traces (Elastic)',
+ name: 'Stack Traces (API)',
  content: (
  <>
  <EuiSpacer />
@@ -57,6 +61,20 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props)
  </>
  ),
  },
+ {
+ id: 'stacktrace-elastic-data',
+ name: 'Stack Traces (Data Plugin)',
+ content: (
+ <>
+ <EuiSpacer />
+ <TopNContext.Provider value={topnData}>
+ <StackTraceNavigation fetchTopN={fetchTopNData} setTopN={setTopNData} />
+ <StackedBarChart id="topn-data" name="topn-data" height={400} x="x" y="y" category="g" />
+ <ChartGrid maximum={10} />
+ </TopNContext.Provider>
+ </>
+ ),
+ },
  {
  id: 'flamegraph-elastic',
  name: 'FlameGraph (Elastic)',

src/plugins/profiling/public/components/contexts/topn.tsx

@@ -8,4 +8,4 @@
 
 import { createContext } from 'react';
 
-export const TopNContext = createContext();
+export const TopNContext = createContext({});

src/plugins/profiling/public/components/stacktrace-nav.tsx

@@ -101,13 +101,19 @@ export const StackTraceNavigation = ({ fetchTopN, setTopN }) => {
  });
 
  console.log(new Date().toISOString(), 'started payload retrieval');
- fetchTopN(topnValue[0].value, dateValue[0].value).then((response) => {
- console.log(new Date().toISOString(), 'finished payload retrieval');
- const samples = getTopN(response);
- const series = groupSamplesByCategory(samples);
- setTopN({ samples, series });
- console.log(new Date().toISOString(), 'updated local state');
- });
+ fetchTopN(topnValue[0].value, dateValue[0].value)
+ .then((response) => {
+ console.log(new Date().toISOString(), 'finished payload retrieval');
+ const samples = getTopN(response);
+ const series = groupSamplesByCategory(samples);
+ console.log('sample %o', samples);
+ console.log('series %o', series);
+ setTopN({ samples, series });
+ console.log(new Date().toISOString(), 'updated local state');
+ })
+ .catch((err) => {
+ console.log('error when reading topN data: ' + err.message);
+ });
  }, [toggleTopNSelected, toggleDateSelected]);
 
  return (

src/plugins/profiling/public/plugin.tsx

@@ -6,18 +6,29 @@
  * Side Public License, v 1.
  */
 
-import { AppMountParameters, CoreSetup, CoreStart, Plugin } from '../../../core/public';
+import { AppMountParameters, CoreSetup, CoreStart, Plugin } from 'kibana/public';
 import { getServices } from './services';
+import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public';
 
-export class ProdfilerPlugin implements Plugin {
- public setup(core: CoreSetup) {
+export interface ProdfilerPluginStartDeps {
+ data: DataPublicPluginStart;
+}
+
+export interface ProdfilerPluginSetupDeps {
+ data: DataPublicPluginSetup;
+}
+
+export class ProdfilerPlugin
+ implements Plugin<void, void, ProdfilerPluginSetupDeps, ProdfilerPluginStartDeps>
+{
+ public setup(core: CoreSetup<ProdfilerPluginStartDeps>) {
  // Register an application into the side navigation menu
  core.application.register({
  id: 'prodfiler',
  title: 'Prodfiler',
  async mount({ element }: AppMountParameters) {
- const [coreStart] = await core.getStartServices();
- const startServices = getServices(coreStart);
+ const [coreStart, dataPlugin] = await core.getStartServices();
+ const startServices = getServices(coreStart, dataPlugin);
  const { renderApp } = await import('./app');
  return renderApp(startServices, element);
  },

src/plugins/profiling/public/services.ts

@@ -8,11 +8,19 @@
 
 import { CoreStart, HttpFetchError, HttpFetchQuery } from 'kibana/public';
 import { getRemoteRoutePaths } from '../common';
+import { ProdfilerPluginStartDeps } from './plugin';
+import {
+ DOWNSAMPLED_TOPN_STRATEGY,
+ DownsampledRequest,
+ DownsampledTopNResponse,
+ TopNAggregateResponse,
+} from '../common/types';
 
 export interface Services {
  fetchTopN: (type: string, seconds: string) => Promise<any[] | HttpFetchError>;
  fetchElasticFlamechart: (seconds: string) => Promise<any[] | HttpFetchError>;
  fetchPixiFlamechart: (seconds: string) => Promise<any[] | HttpFetchError>;
+ fetchTopNData: (searchField: string, seconds: string) => Promise<TopNAggregateResponse>;
 }
 
 function getFetchQuery(seconds: string): HttpFetchQuery {
@@ -27,11 +35,51 @@ function getFetchQuery(seconds: string): HttpFetchQuery {
  } as HttpFetchQuery;
 }
 
-export function getServices(core: CoreStart): Services {
+export function getServices(core: CoreStart, data?: ProdfilerPluginStartDeps): Services {
  // To use local fixtures instead, use getLocalRoutePaths
  const paths = getRemoteRoutePaths();
 
  return {
+ fetchTopNData: async (searchField: string, seconds: string): Promise<TopNAggregateResponse> => {
+ const unixTime = Math.floor(Date.now() / 1000);
+ const response: TopNAggregateResponse = { topN: { histogram: { buckets: [] } } };
+ data!.data.search
+ .search<DownsampledRequest, DownsampledTopNResponse>(
+ {
+ params: {
+ projectID: 5,
+ timeFrom: unixTime - parseInt(seconds, 10),
+ timeTo: unixTime,
+ // FIXME remove hard-coded value for topN items length and expose it through the UI
+ topNItems: 100,
+ searchField,
+ },
+ },
+ {
+ strategy: DOWNSAMPLED_TOPN_STRATEGY,
+ }
+ )
+ .subscribe({
+ next: (result) => {
+ console.log('subscription data plugin rawResponse: %o', result.rawResponse);
+ response.topN.histogram = result.rawResponse.aggregations.histogram;
+ },
+ // TODO error handling
+ error: (err) => {
+ console.log('subscription error: %o', err);
+ },
+ // FIXME remove this, used for debugging only
+ complete: () => {
+ console.log('subscription completed');
+ },
+ });
+
+ console.log('returning Promise of TopNAggregateResponse');
+ return await new Promise<TopNAggregateResponse>((resolve, _) => {
+ return resolve(response);
+ });
+ },
+
  fetchTopN: async (type: string, seconds: string) => {
  try {
  const query = getFetchQuery(seconds);

src/plugins/profiling/server/plugin.ts

@@ -5,12 +5,14 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'kibana/server';
+import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from 'kibana/server';
 
 import type { DataRequestHandlerContext } from '../../data/server';
 
 import { ProfilingPluginSetupDeps, ProfilingPluginStartDeps } from './types';
 import { registerRoutes } from './routes';
+import { DownsampledTopNFactory } from './search/strategy';
+import { DOWNSAMPLED_TOPN_STRATEGY } from '../common/types';
 
 export class ProfilingPlugin
  implements Plugin<void, void, ProfilingPluginSetupDeps, ProfilingPluginStartDeps>
@@ -21,14 +23,18 @@ export class ProfilingPlugin
  this.logger = initializerContext.logger.get();
  }
 
- public setup(core: CoreSetup<ProfilingPluginStartDeps>, { data }: ProfilingPluginSetupDeps) {
+ public setup(core: CoreSetup<ProfilingPluginStartDeps>, deps: ProfilingPluginSetupDeps) {
  this.logger.debug('profiling: Setup');
  // TODO we should create a query here using "data".
  // We should ensure there are profiling data in the expected indices
  // and return an error otherwise.
  // This should be done only once at startup and before exposing the routed APIs.
  const router = core.http.createRouter<DataRequestHandlerContext>();
- core.getStartServices().then(() => {
+ core.getStartServices().then(([_, depsStart]) => {
+ deps.data.search.registerSearchStrategy(
+ DOWNSAMPLED_TOPN_STRATEGY,
+ DownsampledTopNFactory(depsStart.data)
+ );
  registerRoutes(router, this.logger);
  });
 

src/plugins/profiling/server/search/strategy.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types';
+import { IEsSearchRequest, ISearchStrategy, PluginStart } from '../../../data/server';
+import {
+ autoHistogramSumCountOnGroupByField,
+ newProjectTimeQuery,
+ ProjectTimeQuery,
+} from '../routes/mappings';
+import { downsampledIndex, getSampledTraceEventsIndex } from '../routes/search_flamechart';
+import { DownsampledRequest, DownsampledTopNResponse } from '../../common/types';
+
+export const DownsampledTopNFactory = (
+ data: PluginStart
+): ISearchStrategy<DownsampledRequest, DownsampledTopNResponse> => {
+ const es = data.search.getSearchStrategy();
+
+ // FIXME these 2 constants should be configurable?
+ const initialExp = 6;
+ const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
+
+ // Calculate the right down-sampled index to query data from
+ const sampleCountFromInitialExp = (filter: ProjectTimeQuery, options, deps): number => {
+ // By default, we return no samples and use the un-sampled index
+ let sampleCount = 0;
+ es.search(
+ {
+ params: {
+ index: downsampledIndex + initialExp,
+ body: {
+ query: filter,
+ size: 0,
+ track_total_hits: true,
+ },
+ },
+ },
+ options,
+ deps
+ ).subscribe({
+ next: (value) => {
+ sampleCount = (value.rawResponse.hits.total as SearchTotalHits).value;
+ },
+ });
+ return sampleCount;
+ };
+ return {
+ search: (request, options, deps) => {
+ const { projectID, timeFrom, timeTo, topNItems, searchField } = request.params!;
+ const filter = newProjectTimeQuery(
+ projectID.toString(),
+ timeFrom.toString(),
+ timeTo.toString()
+ );
+
+ // Create the query for the actual data
+ const downsampledReq = {
+ params: {
+ index: getSampledTraceEventsIndex(
+ targetSampleSize,
+ sampleCountFromInitialExp(filter, options, deps),
+ initialExp
+ ).name,
+ body: {
+ query: filter,
+ aggs: {
+ histogram: autoHistogramSumCountOnGroupByField(searchField, topNItems),
+ },
+ },
+ },
+ } as IEsSearchRequest;
+ return es.search(downsampledReq, options, deps);
+ },
+ cancel: async (id, options, deps) => {
+ if (es.cancel) {
+ await es.cancel(id, options, deps);
+ }
+ },
+ };
+};