[Lens] Expose Elasticsearch accuracy warnings to the user (#116632)

* [Lens] Expose Elasticsearch accuracy warnings to the user

Closes: #94918

* fix comments

* update text

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Author: 2 people

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -89,6 +89,7 @@ readonly links: {
             readonly range: string;
             readonly significant_terms: string;
             readonly terms: string;
+            readonly terms_doc_count_error: string;
             readonly avg: string;
             readonly avg_bucket: string;
             readonly max_bucket: string;

docs/development/core/public/kibana-plugin-core-public.doclinksstart.md

@@ -118,6 +118,7 @@ export class DocLinksService {
           range: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-range-aggregation.html`,
           significant_terms: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-significantterms-aggregation.html`,
           terms: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-terms-aggregation.html`,
+          terms_doc_count_error: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-terms-aggregation.html#_per_bucket_document_count_error`,
           avg: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-avg-aggregation.html`,
           avg_bucket: `${ELASTICSEARCH_DOCS}search-aggregations-pipeline-avg-bucket-aggregation.html`,
           max_bucket: `${ELASTICSEARCH_DOCS}search-aggregations-pipeline-max-bucket-aggregation.html`,
@@ -613,6 +614,7 @@ export interface DocLinksStart {
       readonly range: string;
       readonly significant_terms: string;
       readonly terms: string;
+      readonly terms_doc_count_error: string;
       readonly avg: string;
       readonly avg_bucket: string;
       readonly max_bucket: string;

src/core/public/doc_links/doc_links_service.ts

@@ -557,6 +557,7 @@ export interface DocLinksStart {
             readonly range: string;
             readonly significant_terms: string;
             readonly terms: string;
+            readonly terms_doc_count_error: string;
             readonly avg: string;
             readonly avg_bucket: string;
             readonly max_bucket: string;

src/core/public/public.api.md

@@ -53,6 +53,7 @@ export interface AggTypeConfig<
   json?: boolean;
   decorateAggConfig?: () => any;
   postFlightRequest?: PostFlightRequestFn<TAggConfig>;
+  hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
   getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat;
   getValue?: (agg: TAggConfig, bucket: any) => any;
   getKey?: (bucket: any, key: any, agg: TAggConfig) => any;
@@ -180,6 +181,9 @@ export class AggType<
    * is created, giving the agg type a chance to modify the agg config
    */
   decorateAggConfig: () => any;
+
+  hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
+
   /**
    * A function that needs to be called after the main request has been made
    * and should return an updated response
@@ -283,6 +287,7 @@ export class AggType<
     this.getResponseAggs = config.getResponseAggs || (() => {});
     this.decorateAggConfig = config.decorateAggConfig || (() => ({}));
     this.postFlightRequest = config.postFlightRequest || identity;
+    this.hasPrecisionError = config.hasPrecisionError;
 
     this.getSerializedFormat =
       config.getSerializedFormat ||

src/plugins/data/common/search/aggs/agg_type.ts

@@ -286,7 +286,19 @@ describe('Terms Agg', () => {
         { typesRegistry: mockAggTypesRegistry() }
       );
       const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+
       expect(params.order).toEqual({ 'test-orderAgg.50': 'desc' });
     });
+
+    test('should override "hasPrecisionError" for the "terms" bucket type', () => {
+      const aggConfigs = getAggConfigs();
+      const { type } = aggConfigs.aggs[0];
+
+      expect(type.hasPrecisionError).toBeInstanceOf(Function);
+
+      expect(type.hasPrecisionError!({})).toBeFalsy();
+      expect(type.hasPrecisionError!({ doc_count_error_upper_bound: 0 })).toBeFalsy();
+      expect(type.hasPrecisionError!({ doc_count_error_upper_bound: -1 })).toBeTruthy();
+    });
   });
 });

src/plugins/data/common/search/aggs/buckets/terms.test.ts

@@ -85,6 +85,7 @@ export const getTermsBucketAgg = () =>
       };
     },
     createFilter: createFilterTerms,
+    hasPrecisionError: (aggBucket) => Boolean(aggBucket?.doc_count_error_upper_bound),
     postFlightRequest: async (
       resp,
       aggConfigs,

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -9,3 +9,4 @@
 export { tabifyDocs, flattenHit } from './tabify_docs';
 export { tabifyAggResponse } from './tabify';
 export { tabifyGetColumns } from './get_columns';
+export { checkColumnForPrecisionError } from './utils';

src/plugins/data/common/search/tabify/index.ts

@@ -166,6 +166,7 @@ describe('TabbedAggResponseWriter class', () => {
           field: 'geo.src',
           source: 'esaggs',
           sourceParams: {
+            hasPrecisionError: false,
             enabled: true,
             id: '1',
             indexPatternId: '1234',
@@ -193,6 +194,7 @@ describe('TabbedAggResponseWriter class', () => {
           },
           source: 'esaggs',
           sourceParams: {
+            hasPrecisionError: false,
             appliedTimeRange: undefined,
             enabled: true,
             id: '2',
@@ -227,6 +229,7 @@ describe('TabbedAggResponseWriter class', () => {
           field: 'geo.src',
           source: 'esaggs',
           sourceParams: {
+            hasPrecisionError: false,
             enabled: true,
             id: '1',
             indexPatternId: '1234',
@@ -254,6 +257,7 @@ describe('TabbedAggResponseWriter class', () => {
           },
           source: 'esaggs',
           sourceParams: {
+            hasPrecisionError: false,
             appliedTimeRange: undefined,
             enabled: true,
             id: '2',

src/plugins/data/common/search/tabify/response_writer.test.ts

@@ -10,7 +10,7 @@ import { isEmpty } from 'lodash';
 import { IAggConfigs } from '../aggs';
 import { tabifyGetColumns } from './get_columns';
 
-import { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
+import type { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
 import { Datatable, DatatableColumn } from '../../../../expressions/common/expression_types/specs';
 
 interface BufferColumn {
@@ -80,6 +80,7 @@ export class TabbedAggResponseWriter {
             params: column.aggConfig.toSerializedFieldFormat(),
             source: 'esaggs',
             sourceParams: {
+              hasPrecisionError: Boolean(column.hasPrecisionError),
               indexPatternId: column.aggConfig.getIndexPattern()?.id,
               appliedTimeRange:
                 column.aggConfig.params.field?.name &&

src/plugins/data/common/search/tabify/response_writer.ts

@@ -42,8 +42,14 @@ export function tabifyAggResponse(
 
       switch (agg.type.type) {
         case AggGroupNames.Buckets:
-          const aggBucket = get(bucket, agg.id);
+          const aggBucket = get(bucket, agg.id) as Record<string, unknown>;
           const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, respOpts?.timeRange);
+          const precisionError = agg.type.hasPrecisionError?.(aggBucket);
+
+          if (precisionError) {
+            // "сolumn" mutation, we have to do this here as this value is filled in based on aggBucket value
+            column.hasPrecisionError = true;
+          }
 
           if (tabifyBuckets.length) {
             tabifyBuckets.forEach((subBucket, tabifyBucketKey) => {

src/plugins/data/common/search/tabify/tabify.ts

@@ -41,6 +41,7 @@ export interface TabbedAggColumn {
   aggConfig: IAggConfig;
   id: string;
   name: string;
+  hasPrecisionError?: boolean;
 }
 
 /** @public **/

src/plugins/data/common/search/tabify/types.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { checkColumnForPrecisionError } from './utils';
+import type { DatatableColumn } from '../../../../expressions';
+
+describe('tabify utils', () => {
+  describe('checkDatatableForPrecisionError', () => {
+    test('should return true if there is a precision error in the column', () => {
+      expect(
+        checkColumnForPrecisionError({
+          meta: {
+            sourceParams: {
+              hasPrecisionError: true,
+            },
+          },
+        } as unknown as DatatableColumn)
+      ).toBeTruthy();
+    });
+    test('should return false if there is no precision error in the column', () => {
+      expect(
+        checkColumnForPrecisionError({
+          meta: {
+            sourceParams: {
+              hasPrecisionError: false,
+            },
+          },
+        } as unknown as DatatableColumn)
+      ).toBeFalsy();
+    });
+    test('should return false if precision error is not defined', () => {
+      expect(
+        checkColumnForPrecisionError({
+          meta: {
+            sourceParams: {},
+          },
+        } as unknown as DatatableColumn)
+      ).toBeFalsy();
+    });
+  });
+});

src/plugins/data/common/search/tabify/utils.test.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { DatatableColumn } from '../../../../expressions';
+
+/** @public **/
+export const checkColumnForPrecisionError = (column: DatatableColumn) =>
+  column.meta.sourceParams?.hasPrecisionError;

src/plugins/data/common/search/tabify/utils.ts

@@ -139,6 +139,7 @@ import {
   // tabify
   tabifyAggResponse,
   tabifyGetColumns,
+  checkColumnForPrecisionError,
 } from '../common';
 
 export { AggGroupLabels, AggGroupNames, METRIC_TYPES, BUCKET_TYPES } from '../common';
@@ -246,6 +247,7 @@ export const search = {
   getResponseInspectorStats,
   tabifyAggResponse,
   tabifyGetColumns,
+  checkColumnForPrecisionError,
 };
 
 /*

src/plugins/data/public/index.ts

@@ -36,6 +36,7 @@ export {
   parseSearchSourceJSON,
   SearchSource,
   SortDirection,
+  checkColumnForPrecisionError,
 } from '../../common/search';
 export type {
   ISessionService,

src/plugins/data/public/search/index.ts

@@ -1 +1 @@
-{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}
+{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}

test/interpreter_functional/snapshots/baseline/combined_test2.json

@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}

test/interpreter_functional/snapshots/baseline/combined_test3.json

@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}