From b8b7edf8ababcbc36e9c3cc26b341f8019109093 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jan 2022 11:57:49 +0100 Subject: [PATCH] [ML] Transforms: Support for terms agg in pivot configurations. (#123634) Adds support for the terms agg for transform pivot configurations. --- .../transform/common/types/pivot_aggs.ts | 1 + .../transform/public/app/common/index.ts | 2 + .../transform/public/app/common/pivot_aggs.ts | 13 +++ .../public/app/hooks/use_pivot_data.ts | 12 ++- .../aggregation_list/popover_form.tsx | 79 +++++++++++++++++-- .../apps/transform/creation_index_pattern.ts | 60 +++++++++++++- 6 files changed, 157 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/transform/common/types/pivot_aggs.ts b/x-pack/plugins/transform/common/types/pivot_aggs.ts index ced4d0a9bce0c..44308940a5870 100644 --- a/x-pack/plugins/transform/common/types/pivot_aggs.ts +++ b/x-pack/plugins/transform/common/types/pivot_aggs.ts @@ -18,6 +18,7 @@ export const PIVOT_SUPPORTED_AGGS = { VALUE_COUNT: 'value_count', FILTER: 'filter', TOP_METRICS: 'top_metrics', + TERMS: 'terms', } as const; export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS]; diff --git a/x-pack/plugins/transform/public/app/common/index.ts b/x-pack/plugins/transform/public/app/common/index.ts index 7081b6db2fe40..7a84ef9c8baa3 100644 --- a/x-pack/plugins/transform/public/app/common/index.ts +++ b/x-pack/plugins/transform/public/app/common/index.ts @@ -39,7 +39,9 @@ export { getEsAggFromAggConfig, isPivotAggsConfigWithUiSupport, isPivotAggsConfigPercentiles, + isPivotAggsConfigTerms, PERCENTILES_AGG_DEFAULT_PERCENTS, + TERMS_AGG_DEFAULT_SIZE, pivotAggsFieldSupport, } from './pivot_aggs'; export type { diff --git a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts index 6f3d3de79c391..42ce5744bcd26 100644 --- a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts +++ b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts @@ -28,6 +28,7 @@ export function isPivotSupportedAggs(arg: unknown): arg is PivotSupportedAggs { } export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99]; +export const TERMS_AGG_DEFAULT_SIZE = 10; export const pivotAggsFieldSupport = { [KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], @@ -45,6 +46,7 @@ export const pivotAggsFieldSupport = { PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER, PIVOT_SUPPORTED_AGGS.TOP_METRICS, + PIVOT_SUPPORTED_AGGS.TERMS, ], [KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], [KBN_FIELD_TYPES.NUMBER]: [ @@ -63,6 +65,7 @@ export const pivotAggsFieldSupport = { PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER, PIVOT_SUPPORTED_AGGS.TOP_METRICS, + PIVOT_SUPPORTED_AGGS.TERMS, ], [KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], [KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], @@ -226,9 +229,15 @@ interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase { percents: number[]; } +interface PivotAggsConfigTerms extends PivotAggsConfigWithUiBase { + agg: typeof PIVOT_SUPPORTED_AGGS.TERMS; + size: number; +} + export type PivotAggsConfigWithUiSupport = | PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles + | PivotAggsConfigTerms | PivotAggsConfigWithExtendedForm; export function isPivotAggsConfigWithUiSupport(arg: unknown): arg is PivotAggsConfigWithUiSupport { @@ -258,6 +267,10 @@ export function isPivotAggsConfigPercentiles(arg: unknown): arg is PivotAggsConf ); } +export function isPivotAggsConfigTerms(arg: unknown): arg is PivotAggsConfigTerms { + return isPopulatedObject(arg, ['agg', 'field', 'size']) && arg.agg === PIVOT_SUPPORTED_AGGS.TERMS; +} + export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport; export type PivotAggsConfigWithUiSupportDict = Dictionary; diff --git a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts index a79b96acd5d7d..300c9c84993a1 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts @@ -13,7 +13,7 @@ import { EuiDataGridColumn } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { getFlattenedObject } from '@kbn/std'; -import { sample, difference } from 'lodash'; +import { difference } from 'lodash'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms'; @@ -79,12 +79,16 @@ export function getCombinedProperties( populatedProperties: PreviewMappingsProperties, docs: Array> ): PreviewMappingsProperties { - // Take a sample from docs and resolve missing mappings - const sampleDoc = sample(docs) ?? {}; - const missingMappings = difference(Object.keys(sampleDoc), Object.keys(populatedProperties)); + // Identify missing mappings + const missingMappings = difference( + // Create an array of unique flattened field names across all docs + [...new Set(docs.flatMap(Object.keys))], + Object.keys(populatedProperties) + ); return { ...populatedProperties, ...missingMappings.reduce((acc, curr) => { + const sampleDoc = docs.find((d) => typeof d[curr] !== 'undefined') ?? {}; acc[curr] = { type: typeof sampleDoc[curr] as ES_FIELD_TYPES }; return acc; }, {} as PreviewMappingsProperties), diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx index 53f2716551289..7370a4fd9287d 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx @@ -13,6 +13,7 @@ import { EuiButton, EuiCodeBlock, EuiComboBox, + EuiFieldNumber, EuiFieldText, EuiForm, EuiFormRow, @@ -32,9 +33,11 @@ import { import { isAggName, isPivotAggsConfigPercentiles, + isPivotAggsConfigTerms, isPivotAggsConfigWithUiSupport, getEsAggFromAggConfig, PERCENTILES_AGG_DEFAULT_PERCENTS, + TERMS_AGG_DEFAULT_SIZE, PivotAggsConfig, PivotAggsConfigWithUiSupportDict, } from '../../../../common'; @@ -75,6 +78,30 @@ function parsePercentsInput(inputValue: string | undefined) { return []; } +// Input string should only include comma separated numbers +function isValidPercentsInput(inputValue: string) { + return /^[0-9]+(,[0-9]+)*$/.test(inputValue); +} + +function getDefaultSize(defaultData: PivotAggsConfig): number | undefined { + if (isPivotAggsConfigTerms(defaultData)) { + return defaultData.size; + } +} + +function parseSizeInput(inputValue: string | undefined) { + if (inputValue !== undefined && isValidSizeInput(inputValue)) { + return parseInt(inputValue, 10); + } + + return TERMS_AGG_DEFAULT_SIZE; +} + +// Input string should only include numbers +function isValidSizeInput(inputValue: string) { + return /^\d+$/.test(inputValue); +} + export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onChange, options }) => { const [aggConfigDef, setAggConfigDef] = useState(cloneDeep(defaultData)); @@ -85,6 +112,9 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha ); const [percents, setPercents] = useState(getDefaultPercents(defaultData)); + const [validPercents, setValidPercents] = useState(agg === PIVOT_SUPPORTED_AGGS.PERCENTILES); + const [size, setSize] = useState(getDefaultSize(defaultData)); + const [validSize, setValidSize] = useState(agg === PIVOT_SUPPORTED_AGGS.TERMS); const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData); @@ -118,10 +148,19 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) { setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS); } + if (aggVal === PIVOT_SUPPORTED_AGGS.TERMS && size === undefined) { + setSize(TERMS_AGG_DEFAULT_SIZE); + } } function updatePercents(inputValue: string) { setPercents(parsePercentsInput(inputValue)); + setValidPercents(isValidPercentsInput(inputValue)); + } + + function updateSize(inputValue: string) { + setSize(parseSizeInput(inputValue)); + setValidSize(isValidSizeInput(inputValue)); } function getUpdatedItem(): PivotAggsConfig { @@ -137,21 +176,29 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha resultField = field[0]; } - if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) { + if (agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) { updatedItem = { - ...aggConfigDef, agg, aggName, field: resultField, dropDownName: defaultData.dropDownName, + percents, + }; + } else if (agg === PIVOT_SUPPORTED_AGGS.TERMS) { + updatedItem = { + agg, + aggName, + field: resultField, + dropDownName: defaultData.dropDownName, + size, }; } else { updatedItem = { + ...aggConfigDef, agg, aggName, field: resultField, dropDownName: defaultData.dropDownName, - percents, }; } @@ -202,13 +249,18 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha percentsText = percents.toString(); } - const validPercents = - agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && parsePercentsInput(percentsText).length > 0; + let sizeText; + if (size !== undefined) { + sizeText = size.toString(); + } let formValid = validAggName; if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) { formValid = validPercents; } + if (formValid && agg === PIVOT_SUPPORTED_AGGS.TERMS) { + formValid = validSize; + } if (isPivotAggsWithExtendedForm(aggConfigDef)) { formValid = validAggName && aggConfigDef.isValid(); } @@ -325,6 +377,23 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha /> )} + {agg === PIVOT_SUPPORTED_AGGS.TERMS && ( + + updateSize(e.target.value)} /> + + )} {isUnsupportedAgg && (