[ML] Kibana API endpoint for histogram chart data

x-pack/plugins/ml/common/constants/field_histograms.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// Default sampler shard size used for field histograms
+export const DEFAULT_SAMPLER_SHARD_SIZE = 5000;

x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx

@@ -20,10 +20,13 @@ import {
   EuiFlexItem,
   EuiSpacer,
   EuiTitle,
+  EuiToolTip,
 } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
 
+import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
+
 import { INDEX_STATUS } from '../../data_frame_analytics/common';
 
 import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
@@ -193,21 +196,31 @@ export const DataGrid: FC<Props> = memo(
               ...(chartsButtonVisible
                 ? {
                     additionalControls: (
-                      <EuiButtonEmpty
-                        aria-checked={chartsVisible}
-                        className={`euiDataGrid__controlBtn${
-                          chartsVisible ? ' euiDataGrid__controlBtn--active' : ''
-                        }`}
-                        data-test-subj={`${dataTestSubj}HistogramButton`}
-                        size="xs"
-                        iconType="visBarVertical"
-                        color="text"
-                        onClick={toggleChartVisibility}
-                      >
-                        {i18n.translate('xpack.ml.dataGrid.histogramButtonText', {
-                          defaultMessage: 'Histogram charts',
+                      <EuiToolTip
+                        content={i18n.translate('xpack.ml.dataGrid.histogramButtonToolTipContent', {
+                          defaultMessage:
+                            'Queries run to fetch histogram chart data will use a sample size per shard of {samplerShardSize} documents.',
+                          values: {
+                            samplerShardSize: DEFAULT_SAMPLER_SHARD_SIZE,
+                          },
                         })}
-                      </EuiButtonEmpty>
+                      >
+                        <EuiButtonEmpty
+                          aria-checked={chartsVisible}
+                          className={`euiDataGrid__controlBtn${
+                            chartsVisible ? ' euiDataGrid__controlBtn--active' : ''
+                          }`}
+                          data-test-subj={`${dataTestSubj}HistogramButton`}
+                          size="xs"
+                          iconType="visBarVertical"
+                          color="text"
+                          onClick={toggleChartVisibility}
+                        >
+                          {i18n.translate('xpack.ml.dataGrid.histogramButtonText', {
+                            defaultMessage: 'Histogram charts',
+                          })}
+                        </EuiButtonEmpty>
+                      </EuiToolTip>
                     ),
                   }
                 : {}),

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -12,7 +12,7 @@ export {
   showDataGridColumnChartErrorMessageToast,
   useRenderCellValue,
 } from './common';
-export { fetchChartsData, ChartData } from './use_column_chart';
+export { getFieldType, ChartData } from './use_column_chart';
 export { useDataGrid } from './use_data_grid';
 export { DataGrid } from './data_grid';
 export {

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getFieldType } from './use_column_chart';
+
+describe('getFieldType()', () => {
+  it('should return the Kibana field type for a given EUI data grid schema', () => {
+    expect(getFieldType('text')).toBe('string');
+    expect(getFieldType('datetime')).toBe('date');
+    expect(getFieldType('numeric')).toBe('number');
+    expect(getFieldType('boolean')).toBe('boolean');
+    expect(getFieldType('json')).toBe('object');
+    expect(getFieldType('non-aggregatable')).toBe(undefined);
+  });
+});

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx

@@ -16,8 +16,6 @@ import { i18n } from '@kbn/i18n';
 
 import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 
-import { stringHash } from '../../../../common/util/string_utils';
-
 import { NON_AGGREGATABLE } from './common';
 
 export const hoveredRow$ = new BehaviorSubject<any | null>(null);
@@ -40,7 +38,7 @@ const getXScaleType = (kbnFieldType: KBN_FIELD_TYPES | undefined): XScaleType =>
   }
 };
 
-const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => {
+export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => {
   if (schema === NON_AGGREGATABLE) {
     return undefined;
   }
@@ -67,188 +65,6 @@ const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | un
   return fieldType;
 };
 
-interface NumericColumnStats {
-  interval: number;
-  min: number;
-  max: number;
-}
-type NumericColumnStatsMap = Record<string, NumericColumnStats>;
-const getAggIntervals = async (
-  indexPatternTitle: string,
-  esSearch: (payload: any) => Promise<any>,
-  query: any,
-  columnTypes: EuiDataGridColumn[]
-): Promise<NumericColumnStatsMap> => {
-  const numericColumns = columnTypes.filter((cT) => {
-    const fieldType = getFieldType(cT.schema);
-    return fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE;
-  });
-
-  if (numericColumns.length === 0) {
-    return {};
-  }
-
-  const minMaxAggs = numericColumns.reduce((aggs, c) => {
-    const id = stringHash(c.id);
-    aggs[id] = {
-      stats: {
-        field: c.id,
-      },
-    };
-    return aggs;
-  }, {} as Record<string, object>);
-
-  const respStats = await esSearch({
-    index: indexPatternTitle,
-    size: 0,
-    body: {
-      query,
-      aggs: minMaxAggs,
-      size: 0,
-    },
-  });
-
-  return Object.keys(respStats.aggregations).reduce((p, aggName) => {
-    const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max];
-    if (!stats.includes(null)) {
-      const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min;
-
-      let aggInterval = 1;
-
-      if (delta > MAX_CHART_COLUMNS) {
-        aggInterval = Math.round(delta / MAX_CHART_COLUMNS);
-      }
-
-      if (delta <= 1) {
-        aggInterval = delta / MAX_CHART_COLUMNS;
-      }
-
-      p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] };
-    }
-
-    return p;
-  }, {} as NumericColumnStatsMap);
-};
-
-interface AggHistogram {
-  histogram: {
-    field: string;
-    interval: number;
-  };
-}
-
-interface AggCardinality {
-  cardinality: {
-    field: string;
-  };
-}
-
-interface AggTerms {
-  terms: {
-    field: string;
-    size: number;
-  };
-}
-
-type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms;
-
-export const fetchChartsData = async (
-  indexPatternTitle: string,
-  esSearch: (payload: any) => Promise<any>,
-  query: any,
-  columnTypes: EuiDataGridColumn[]
-): Promise<ChartData[]> => {
-  const aggIntervals = await getAggIntervals(indexPatternTitle, esSearch, query, columnTypes);
-
-  const chartDataAggs = columnTypes.reduce((aggs, c) => {
-    const fieldType = getFieldType(c.schema);
-    const id = stringHash(c.id);
-    if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) {
-      if (aggIntervals[id] !== undefined) {
-        aggs[`${id}_histogram`] = {
-          histogram: {
-            field: c.id,
-            interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1,
-          },
-        };
-      }
-    } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) {
-      if (fieldType === KBN_FIELD_TYPES.STRING) {
-        aggs[`${id}_cardinality`] = {
-          cardinality: {
-            field: c.id,
-          },
-        };
-      }
-      aggs[`${id}_terms`] = {
-        terms: {
-          field: c.id,
-          size: MAX_CHART_COLUMNS,
-        },
-      };
-    }
-    return aggs;
-  }, {} as Record<string, ChartRequestAgg>);
-
-  if (Object.keys(chartDataAggs).length === 0) {
-    return [];
-  }
-
-  const respChartsData = await esSearch({
-    index: indexPatternTitle,
-    size: 0,
-    body: {
-      query,
-      aggs: chartDataAggs,
-      size: 0,
-    },
-  });
-
-  const chartsData: ChartData[] = columnTypes.map(
-    (c): ChartData => {
-      const fieldType = getFieldType(c.schema);
-      const id = stringHash(c.id);
-
-      if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) {
-        if (aggIntervals[id] === undefined) {
-          return {
-            type: 'numeric',
-            data: [],
-            interval: 0,
-            stats: [0, 0],
-            id: c.id,
-          };
-        }
-
-        return {
-          data: respChartsData.aggregations[`${id}_histogram`].buckets,
-          interval: aggIntervals[id].interval,
-          stats: [aggIntervals[id].min, aggIntervals[id].max],
-          type: 'numeric',
-          id: c.id,
-        };
-      } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) {
-        return {
-          type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean',
-          cardinality:
-            fieldType === KBN_FIELD_TYPES.STRING
-              ? respChartsData.aggregations[`${id}_cardinality`].value
-              : 2,
-          data: respChartsData.aggregations[`${id}_terms`].buckets,
-          id: c.id,
-        };
-      }
-
-      return {
-        type: 'unsupported',
-        id: c.id,
-      };
-    }
-  );
-
-  return chartsData;
-};
-
 interface NumericDataItem {
   key: number;
   key_as_string?: string;

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -11,8 +11,11 @@ import { EuiDataGridColumn } from '@elastic/eui';
 import { CoreSetup } from 'src/core/public';
 
 import { IndexPattern } from '../../../../../../../../../src/plugins/data/public';
+
+import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
+
 import {
-  fetchChartsData,
+  getFieldType,
   getDataGridSchemaFromKibanaFieldType,
   getFieldsFromKibanaIndexPattern,
   showDataGridColumnChartErrorMessageToast,
@@ -105,11 +108,15 @@ export const useIndexData = (
 
   const fetchColumnChartsData = async function () {
     try {
-      const columnChartsData = await fetchChartsData(
-        indexPattern.title,
-        ml.esSearch,
-        query,
-        columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+      const dataLoader = new DataLoader(indexPattern, toastNotifications);
+      const columnChartsData = await dataLoader.loadFieldHistograms(
+        columns
+          .filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+          .map((cT) => ({
+            fieldName: cT.id,
+            type: getFieldType(cT.schema),
+          })),
+        query
       );
       dataGrid.setColumnCharts(columnChartsData);
     } catch (e) {

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts

@@ -12,16 +12,17 @@ import { CoreSetup } from 'src/core/public';
 
 import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
 
+import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader';
+
 import {
-  fetchChartsData,
   getDataGridSchemasFromFieldTypes,
+  getFieldType,
   showDataGridColumnChartErrorMessageToast,
   useDataGrid,
   useRenderCellValue,
   UseIndexDataReturnType,
 } from '../../../../../components/data_grid';
 import { SavedSearchQuery } from '../../../../../contexts/ml';
-import { ml } from '../../../../../services/ml_api_service';
 
 import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common';
 import {
@@ -74,12 +75,16 @@ export const useExplorationResults = (
 
   const fetchColumnChartsData = async function () {
     try {
-      if (jobConfig !== undefined) {
-        const columnChartsData = await fetchChartsData(
-          jobConfig.dest.index,
-          ml.esSearch,
-          searchQuery,
-          columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+      if (jobConfig !== undefined && indexPattern !== undefined) {
+        const dataLoader = new DataLoader(indexPattern, toastNotifications);
+        const columnChartsData = await dataLoader.loadFieldHistograms(
+          columns
+            .filter((cT) => dataGrid.visibleColumns.includes(cT.id))
+            .map((cT) => ({
+              fieldName: cT.id,
+              type: getFieldType(cT.schema),
+            })),
+          searchQuery
         );
         dataGrid.setColumnCharts(columnChartsData);
       }