[ML] Kibana API endpoint for histogram chart data (#70976) (#71619)

- Introduces dedicated Kibana API endpoints as part of ML and transform plugin API endpoints and moves the logic to query and transform the required data from client to server.
- Adds support for sampling to retrieve the data for the field histograms. For now this is not configurable by the end user and is hard coded to 5000. This is to have a first iteration of this functionality in for 7.9 and protect users when querying large clusters. The button to enable the histogram charts now includes a tooltip that mentions the sampler.

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ml/common/constants/field_histograms.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// Default sampler shard size used for field histograms
+export const DEFAULT_SAMPLER_SHARD_SIZE = 5000;

x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx

@@ -20,10 +20,13 @@ import {
  EuiFlexItem,
  EuiSpacer,
  EuiTitle,
+ EuiToolTip,
 } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
 
+import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
+
 import { INDEX_STATUS } from '../../data_frame_analytics/common';
 
 import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
@@ -193,21 +196,31 @@ export const DataGrid: FC<Props> = memo(
  ...(chartsButtonVisible
  ? {
  additionalControls: (
- <EuiButtonEmpty
- aria-checked={chartsVisible}
- className={`euiDataGrid__controlBtn${
- chartsVisible ? ' euiDataGrid__controlBtn--active' : ''
- }`}
- data-test-subj={`${dataTestSubj}HistogramButton`}
- size="xs"
- iconType="visBarVertical"
- color="text"
- onClick={toggleChartVisibility}
- >
- {i18n.translate('xpack.ml.dataGrid.histogramButtonText', {
- defaultMessage: 'Histogram charts',
+ <EuiToolTip
+ content={i18n.translate('xpack.ml.dataGrid.histogramButtonToolTipContent', {
+ defaultMessage:
+ 'Queries run to fetch histogram chart data will use a sample size per shard of {samplerShardSize} documents.',
+ values: {
+ samplerShardSize: DEFAULT_SAMPLER_SHARD_SIZE,
+ },
  })}
- </EuiButtonEmpty>
+ >
+ <EuiButtonEmpty
+ aria-checked={chartsVisible}
+ className={`euiDataGrid__controlBtn${
+ chartsVisible ? ' euiDataGrid__controlBtn--active' : ''
+ }`}
+ data-test-subj={`${dataTestSubj}HistogramButton`}
+ size="xs"
+ iconType="visBarVertical"
+ color="text"
+ onClick={toggleChartVisibility}
+ >
+ {i18n.translate('xpack.ml.dataGrid.histogramButtonText', {
+ defaultMessage: 'Histogram charts',
+ })}
+ </EuiButtonEmpty>
+ </EuiToolTip>
  ),
  }
  : {}),

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -12,7 +12,7 @@ export {
  showDataGridColumnChartErrorMessageToast,
  useRenderCellValue,
 } from './common';
-export { fetchChartsData, ChartData } from './use_column_chart';
+export { getFieldType, ChartData } from './use_column_chart';
 export { useDataGrid } from './use_data_grid';
 export { DataGrid } from './data_grid';
 export {

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getFieldType } from './use_column_chart';
+
+describe('getFieldType()', () => {
+ it('should return the Kibana field type for a given EUI data grid schema', () => {
+ expect(getFieldType('text')).toBe('string');
+ expect(getFieldType('datetime')).toBe('date');
+ expect(getFieldType('numeric')).toBe('number');
+ expect(getFieldType('boolean')).toBe('boolean');
+ expect(getFieldType('json')).toBe('object');
+ expect(getFieldType('non-aggregatable')).toBe(undefined);
+ });
+});

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx

@@ -16,8 +16,6 @@ import { i18n } from '@kbn/i18n';
 
 import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 
-import { stringHash } from '../../../../common/util/string_utils';
-
 import { NON_AGGREGATABLE } from './common';
 
 export const hoveredRow$ = new BehaviorSubject<any | null>(null);
@@ -40,7 +38,7 @@ const getXScaleType = (kbnFieldType: KBN_FIELD_TYPES | undefined): XScaleType =>
  }
 };
 
-const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => {
+export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => {
  if (schema === NON_AGGREGATABLE) {
  return undefined;
  }
@@ -67,188 +65,6 @@ const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | un
  return fieldType;
 };
 
-interface NumericColumnStats {
- interval: number;
- min: number;
- max: number;
-}
-type NumericColumnStatsMap = Record<string, NumericColumnStats>;
-const getAggIntervals = async (
- indexPatternTitle: string,
- esSearch: (payload: any) => Promise<any>,
- query: any,
- columnTypes: EuiDataGridColumn[]
-): Promise<NumericColumnStatsMap> => {
- const numericColumns = columnTypes.filter((cT) => {
- const fieldType = getFieldType(cT.schema);
- return fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE;
- });
-
- if (numericColumns.length === 0) {
- return {};
- }
-
- const minMaxAggs = numericColumns.reduce((aggs, c) => {
- const id = stringHash(c.id);
- aggs[id] = {
- stats: {
- field: c.id,
- },
- };
- return aggs;
- }, {} as Record<string, object>);
-
- const respStats = await esSearch({
- index: indexPatternTitle,
- size: 0,
- body: {
- query,
- aggs: minMaxAggs,
- size: 0,
- },
- });
-
- return Object.keys(respStats.aggregations).reduce((p, aggName) => {
- const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max];
- if (!stats.includes(null)) {
- const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min;
-
- let aggInterval = 1;
-
- if (delta > MAX_CHART_COLUMNS) {
- aggInterval = Math.round(delta / MAX_CHART_COLUMNS);
- }
-
- if (delta <= 1) {
- aggInterval = delta / MAX_CHART_COLUMNS;
- }
-
- p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] };
- }
-
- return p;
- }, {} as NumericColumnStatsMap);
-};
-
-interface AggHistogram {
- histogram: {
- field: string;
- interval: number;
- };
-}
-
-interface AggCardinality {
- cardinality: {
- field: string;
- };
-}
-
-interface AggTerms {
- terms: {
- field: string;
- size: number;
- };
-}
-
-type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms;
-
-export const fetchChartsData = async (
- indexPatternTitle: string,
- esSearch: (payload: any) => Promise<any>,
- query: any,
- columnTypes: EuiDataGridColumn[]
-): Promise<ChartData[]> => {
- const aggIntervals = await getAggIntervals(indexPatternTitle, esSearch, query, columnTypes);
-
- const chartDataAggs = columnTypes.reduce((aggs, c) => {
- const fieldType = getFieldType(c.schema);
- const id = stringHash(c.id);
- if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) {
- if (aggIntervals[id] !== undefined) {
- aggs[`${id}_histogram`] = {
- histogram: {
- field: c.id,
- interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1,
- },
- };
- }
- } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) {
- if (fieldType === KBN_FIELD_TYPES.STRING) {
- aggs[`${id}_cardinality`] = {
- cardinality: {
- field: c.id,
- },
- };
- }
- aggs[`${id}_terms`] = {
- terms: {
- field: c.id,
- size: MAX_CHART_COLUMNS,
- },
- };
- }
- return aggs;
- }, {} as Record<string, ChartRequestAgg>);
-
- if (Object.keys(chartDataAggs).length === 0) {
- return [];
- }
-
- const respChartsData = await esSearch({
- index: indexPatternTitle,
- size: 0,
- body: {
- query,
- aggs: chartDataAggs,
- size: 0,
- },
- });
-
- const chartsData: ChartData[] = columnTypes.map(
- (c): ChartData => {
- const fieldType = getFieldType(c.schema);
- const id = stringHash(c.id);
-
- if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) {
- if (aggIntervals[id] === undefined) {
- return {
- type: 'numeric',
- data: [],
- interval: 0,
- stats: [0, 0],
- id: c.id,
- };
- }
-
- return {
- data: respChartsData.aggregations[`${id}_histogram`].buckets,
- interval: aggIntervals[id].interval,
- stats: [aggIntervals[id].min, aggIntervals[id].max],
- type: 'numeric',
- id: c.id,
- };
- } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) {
- return {
- type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean',
- cardinality:
- fieldType === KBN_FIELD_TYPES.STRING
- ? respChartsData.aggregations[`${id}_cardinality`].value
- : 2,
- data: respChartsData.aggregations[`${id}_terms`].buckets,
- id: c.id,
- };
- }
-
- return {
- type: 'unsupported',
- id: c.id,
- };
- }
- );
-
- return chartsData;
-};
-
 interface NumericDataItem {
  key: number;
  key_as_string?: string;

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -4,15 +4,18 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
 
 import { EuiDataGridColumn } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
 
 import { IndexPattern } from '../../../../../../../../../src/plugins/data/public';
+
+import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
+
 import {
- fetchChartsData,
+ getFieldType,
  getDataGridSchemaFromKibanaFieldType,
  getFieldsFromKibanaIndexPattern,
  showDataGridColumnChartErrorMessageToast,
@@ -103,13 +106,20 @@ export const useIndexData = (
  // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]);
 
+ const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [
+ indexPattern,
+ ]);
+
  const fetchColumnChartsData = async function () {
  try {
- const columnChartsData = await fetchChartsData(
- indexPattern.title,
- ml.esSearch,
- query,
- columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+ const columnChartsData = await dataLoader.loadFieldHistograms(
+ columns
+ .filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+ .map((cT) => ({
+ fieldName: cT.id,
+ type: getFieldType(cT.schema),
+ })),
+ query
  );
  dataGrid.setColumnCharts(columnChartsData);
  } catch (e) {