[8.14] [ML] Fix runtime fields occasionally cause redundant refetch o…

…f data in Index data visualizer (elastic#181853) (elastic#181906) # Backport This will backport the following commits from `main` to `8.14`: - [[ML] Fix runtime fields occasionally cause redundant refetch of data in Index data visualizer (elastic#181853)](elastic#181853)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <43350163+qn895@users.noreply.github.com>
mgiota · Apr 26, 2024 · 5cee001 · 5cee001
1 parent e5cf41a
commit 5cee001
Showing 1 changed file with 36 additions and 17 deletions.
diff --git a/...ugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/...ugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts
@@ -19,6 +19,7 @@ import { getProcessedFields } from '@kbn/ml-data-grid';
 import { buildBaseFilterCriteria } from '@kbn/ml-query-utils';
 import { isDefined } from '@kbn/ml-is-defined';
 import type { FieldSpec } from '@kbn/data-views-plugin/common';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { useDataVisualizerKibana } from '../../kibana_context';
 import type {
   AggregatableFieldOverallStats,
@@ -51,6 +52,18 @@ import {
   fetchDataWithTimeout,
   rateLimitingForkJoin,
 } from '../search_strategy/requests/fetch_utils';
+
+const getPopulatedFieldsInIndex = (
+  populatedFieldsInIndexWithoutRuntimeFields: Set<string> | undefined | null,
+  runtimeFieldMap: MappingRuntimeFields | undefined
+): Set<string> | undefined | null => {
+  if (!populatedFieldsInIndexWithoutRuntimeFields) return undefined;
+  const runtimeFields = runtimeFieldMap ? Object.keys(runtimeFieldMap) : undefined;
+  return runtimeFields && runtimeFields?.length > 0
+    ? new Set([...Array.from(populatedFieldsInIndexWithoutRuntimeFields), ...runtimeFields])
+    : populatedFieldsInIndexWithoutRuntimeFields;
+};
+
 export function useOverallStats<TParams extends OverallStatsSearchStrategyParams>(
   esql = false,
   searchStrategyParams: TParams | undefined,
@@ -68,7 +81,7 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
   } = useDataVisualizerKibana();
 
   const [stats, setOverallStats] = useState<OverallStats>(getDefaultPageState().overallStats);
-  const [populatedFieldsInIndex, setPopulatedFieldsInIndex] = useState<
+  const [populatedFieldsInIndexWithoutRuntimeFields, setPopulatedFieldsInIndex] = useState<
     | Set<string>
     // request to fields caps has not been made yet
     | undefined
@@ -92,10 +105,9 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
 
       // If null, that means we tried to fetch populated fields already but it timed out
       // so don't try again
-      if (!searchStrategyParams || populatedFieldsInIndex === null) return;
+      if (!searchStrategyParams || populatedFieldsInIndexWithoutRuntimeFields === null) return;
 
-      const { index, searchQuery, timeFieldName, earliest, latest, runtimeFieldMap } =
-        searchStrategyParams;
+      const { index, searchQuery, timeFieldName, earliest, latest } = searchStrategyParams;
 
       const fetchPopulatedFields = async () => {
         populatedFieldsAbortCtrl.current.abort();
@@ -131,15 +143,7 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
 
         if (!unmounted) {
           if (Array.isArray(nonEmptyFields)) {
-            setPopulatedFieldsInIndex(
-              new Set([
-                ...nonEmptyFields.map((field) => field.name),
-                // Field caps API don't know about runtime fields
-                // so by default we expect runtime fields to be populated
-                // so we can later check as needed
-                ...Object.keys(runtimeFieldMap ?? {}),
-              ])
-            );
+            setPopulatedFieldsInIndex(new Set([...nonEmptyFields.map((field) => field.name)]));
           } else {
             setPopulatedFieldsInIndex(null);
           }
@@ -154,25 +158,33 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
     },
     // eslint-disable-next-line react-hooks/exhaustive-deps
     [
-      data.dataViews,
       searchStrategyParams?.timeFieldName,
       searchStrategyParams?.earliest,
       searchStrategyParams?.latest,
       searchStrategyParams?.searchQuery,
       searchStrategyParams?.index,
-      searchStrategyParams?.runtimeFieldMap,
     ]
   );
+
   const startFetch = useCallback(async () => {
     try {
       searchSubscription$.current?.unsubscribe();
       abortCtrl.current.abort();
       abortCtrl.current = new AbortController();
 
-      if (!searchStrategyParams || lastRefresh === 0 || populatedFieldsInIndex === undefined) {
+      if (
+        !searchStrategyParams ||
+        lastRefresh === 0 ||
+        populatedFieldsInIndexWithoutRuntimeFields === undefined
+      ) {
         return;
       }
 
+      const populatedFieldsInIndex = getPopulatedFieldsInIndex(
+        populatedFieldsInIndexWithoutRuntimeFields,
+        searchStrategyParams.runtimeFieldMap
+      );
+
       setFetchState({
         ...getInitialProgress(),
         isRunning: true,
@@ -391,7 +403,14 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
         displayError(toasts, searchStrategyParams!.index, extractErrorProperties(error));
       }
     }
-  }, [data, searchStrategyParams, toasts, lastRefresh, probability, populatedFieldsInIndex]);
+  }, [
+    data,
+    searchStrategyParams,
+    toasts,
+    lastRefresh,
+    probability,
+    populatedFieldsInIndexWithoutRuntimeFields,
+  ]);
 
   const cancelFetch = useCallback(() => {
     searchSubscription$.current?.unsubscribe();