From 1d49166203ff0137a377b27be994b6ba76d79893 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Thu, 7 Jan 2021 00:28:05 +0100 Subject: [PATCH] [Transform] Fix transform preview for the latest function (#87168) * [Transform] retrieve mappings from the source index * [Transform] fix preview for nested props * [Transform] exclude meta fields * [Transform] use agg config to only suggest term agg supported fields * [Transform] refactor * [Transform] remove incorrect data mock Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../transform/common/types/transform.ts | 4 +- .../transform/common/utils/object_utils.ts | 27 +++++++-- .../public/app/__mocks__/app_dependencies.tsx | 2 +- .../hooks/use_latest_function_config.ts | 56 +++++++++++-------- .../transform/server/routes/api/transforms.ts | 30 +++++++++- 5 files changed, 86 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/transform/common/types/transform.ts b/x-pack/plugins/transform/common/types/transform.ts index 8ebbfe6e70944..d4852f0144539 100644 --- a/x-pack/plugins/transform/common/types/transform.ts +++ b/x-pack/plugins/transform/common/types/transform.ts @@ -49,9 +49,7 @@ export function isPivotTransform( return transform.hasOwnProperty('pivot'); } -export function isLatestTransform( - transform: TransformBaseConfig -): transform is TransformLatestConfig { +export function isLatestTransform(transform: any): transform is TransformLatestConfig { return transform.hasOwnProperty('latest'); } diff --git a/x-pack/plugins/transform/common/utils/object_utils.ts b/x-pack/plugins/transform/common/utils/object_utils.ts index f21460e08098d..8dd017fdaa48e 100644 --- a/x-pack/plugins/transform/common/utils/object_utils.ts +++ b/x-pack/plugins/transform/common/utils/object_utils.ts @@ -6,17 +6,32 @@ // This is similar to lodash's get() except that it's TypeScript aware and is able to infer return types. // It splits the attribute key string and uses reduce with an idx check to access nested attributes. -export const getNestedProperty = ( +export function getNestedProperty( obj: Record, accessor: string, defaultValue?: any -) => { - const value = accessor.split('.').reduce((o, i) => o?.[i], obj); +): any { + const accessorKeys = accessor.split('.'); - if (value === undefined) return defaultValue; + let o = obj; + for (let i = 0; i < accessorKeys.length; i++) { + const keyPart = accessorKeys[i]; + o = o?.[keyPart]; + if (Array.isArray(o)) { + o = o.map((v) => + typeof v === 'object' + ? // from this point we need to resolve path for each element in the collection + getNestedProperty(v, accessorKeys.slice(i + 1, accessorKeys.length).join('.')) + : v + ); + break; + } + } - return value; -}; + if (o === undefined) return defaultValue; + + return o; +} export const setNestedProperty = (obj: Record, accessor: string, value: any) => { let ref = obj; diff --git a/x-pack/plugins/transform/public/app/__mocks__/app_dependencies.tsx b/x-pack/plugins/transform/public/app/__mocks__/app_dependencies.tsx index 53297cd7d2661..11d6daf01b3b8 100644 --- a/x-pack/plugins/transform/public/app/__mocks__/app_dependencies.tsx +++ b/x-pack/plugins/transform/public/app/__mocks__/app_dependencies.tsx @@ -31,7 +31,7 @@ const appDependencies = { export const useAppDependencies = () => { const ml = useContext(MlSharedContext); - return { ...appDependencies, ml, savedObjects: jest.fn(), data: jest.fn() }; + return { ...appDependencies, ml, savedObjects: jest.fn() }; }; export const useToastNotifications = () => { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts index 53f38a7bad44e..7df6b11dc27ec 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts @@ -11,6 +11,8 @@ import { LatestFunctionConfigUI } from '../../../../../../../common/types/transf import { StepDefineFormProps } from '../step_define_form'; import { StepDefineExposedState } from '../common'; import { LatestFunctionConfig } from '../../../../../../../common/api_schemas/transforms'; +import { AggConfigs, FieldParamType } from '../../../../../../../../../../src/plugins/data/common'; +import { useAppDependencies } from '../../../../../app_dependencies'; /** * Latest function config mapper between API and UI @@ -32,26 +34,33 @@ export const latestConfigMapper = { /** * Provides available options for unique_key and sort fields * @param indexPattern + * @param aggConfigs */ -function getOptions(indexPattern: StepDefineFormProps['searchItems']['indexPattern']) { - const uniqueKeyOptions: Array> = []; - const sortFieldOptions: Array> = []; - - const ignoreFieldNames = new Set(['_id', '_index', '_type']); - - for (const field of indexPattern.fields) { - if (ignoreFieldNames.has(field.name)) { - continue; - } - - if (field.aggregatable) { - uniqueKeyOptions.push({ label: field.displayName, value: field.name }); - } - - if (field.sortable) { - sortFieldOptions.push({ label: field.displayName, value: field.name }); - } - } +function getOptions( + indexPattern: StepDefineFormProps['searchItems']['indexPattern'], + aggConfigs: AggConfigs +) { + const aggConfig = aggConfigs.aggs[0]; + const param = aggConfig.type.params.find((p) => p.type === 'field'); + const filteredIndexPatternFields = param + ? ((param as unknown) as FieldParamType).getAvailableFields(aggConfig) + : []; + + const ignoreFieldNames = new Set(['_source', '_type', '_index', '_id', '_version', '_score']); + + const uniqueKeyOptions: Array> = filteredIndexPatternFields + .filter((v) => !ignoreFieldNames.has(v.name)) + .map((v) => ({ + label: v.displayName, + value: v.name, + })); + + const sortFieldOptions: Array> = indexPattern.fields + .filter((v) => !ignoreFieldNames.has(v.name) && v.sortable) + .map((v) => ({ + label: v.displayName, + value: v.name, + })); return { uniqueKeyOptions, sortFieldOptions }; } @@ -92,9 +101,12 @@ export function useLatestFunctionConfig( sort: defaults.sort, }); - const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => getOptions(indexPattern), [ - indexPattern, - ]); + const { data } = useAppDependencies(); + + const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => { + const aggConfigs = data.search.aggs.createAggConfigs(indexPattern, [{ type: 'terms' }]); + return getOptions(indexPattern, aggConfigs); + }, [indexPattern, data.search.aggs]); const updateLatestFunctionConfig = useCallback( (update) => diff --git a/x-pack/plugins/transform/server/routes/api/transforms.ts b/x-pack/plugins/transform/server/routes/api/transforms.ts index fd93e3055eab5..07e06ec3af3db 100644 --- a/x-pack/plugins/transform/server/routes/api/transforms.ts +++ b/x-pack/plugins/transform/server/routes/api/transforms.ts @@ -57,6 +57,7 @@ import { addBasePath } from '../index'; import { isRequestTimeout, fillResultsWithTimeouts, wrapError, wrapEsError } from './error_utils'; import { registerTransformsAuditMessagesRoutes } from './transforms_audit_messages'; import { IIndexPattern } from '../../../../../../src/plugins/data/common/index_patterns'; +import { isLatestTransform } from '../../../common/types/transform'; enum TRANSFORM_ACTIONS { STOP = 'stop', @@ -531,9 +532,36 @@ const previewTransformHandler: RequestHandler< PostTransformsPreviewRequestSchema > = async (ctx, req, res) => { try { + const reqBody = req.body; const { body } = await ctx.core.elasticsearch.client.asCurrentUser.transform.previewTransform({ - body: req.body, + body: reqBody, }); + if (isLatestTransform(reqBody)) { + // for the latest transform mappings properties have to be retrieved from the source + const fieldCapsResponse = await ctx.core.elasticsearch.client.asCurrentUser.fieldCaps({ + index: reqBody.source.index, + fields: '*', + include_unmapped: false, + }); + + const fieldNamesSet = new Set(Object.keys(fieldCapsResponse.body.fields)); + + const fields = Object.entries( + fieldCapsResponse.body.fields as Record> + ).reduce((acc, [fieldName, fieldCaps]) => { + const fieldDefinition = Object.values(fieldCaps)[0]; + const isMetaField = fieldDefinition.type.startsWith('_') || fieldName === '_doc_count'; + const isKeywordDuplicate = + fieldName.endsWith('.keyword') && fieldNamesSet.has(fieldName.split('.keyword')[0]); + if (isMetaField || isKeywordDuplicate) { + return acc; + } + acc[fieldName] = { ...fieldDefinition }; + return acc; + }, {} as Record); + + body.generated_dest_index.mappings.properties = fields; + } return res.ok({ body }); } catch (e) { return res.customError(wrapError(wrapEsError(e)));