From 772ebacd37ba9cde05c5b28bde94b010fd7c7ad3 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 18 Sep 2020 10:21:52 -0600 Subject: [PATCH] [ML] add geo point combined field to CSV import (#77117) * [ML] add geo point combined field to CSV import * remove some geo_point specific logic * Account for properties layer in find_file_structure mappings * improve checking of name collision to include combined fields and mappings * add delete button * fix function name * fill in unknowns with defined types * tslint changes * get tslint passing * show readonly combined fields in simple tab * handle column_names being undefined * add unit tests for modifying mappings and pipeline * review feedback * do not change combinedFields on reset Co-authored-by: Elastic Machine --- .../ml/common/types/file_datavisualizer.ts | 4 +- .../combined_fields/combined_field_label.tsx | 20 ++ .../combined_fields/combined_fields_form.tsx | 237 ++++++++++++++++++ .../combined_fields_read_only_form.tsx | 36 +++ .../components/combined_fields/geo_point.tsx | 189 ++++++++++++++ .../components/combined_fields/index.ts | 15 ++ .../components/combined_fields/types.ts | 12 + .../components/combined_fields/utils.test.ts | 235 +++++++++++++++++ .../components/combined_fields/utils.ts | 174 +++++++++++++ .../components/import_settings/advanced.tsx | 19 ++ .../import_settings/import_settings.tsx | 12 + .../components/import_settings/simple.tsx | 7 + .../components/import_view/import_view.js | 29 ++- 13 files changed, 986 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_field_label.tsx create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_read_only_form.tsx create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/index.ts create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/types.ts create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.test.ts create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index a8b775c8d5f609..9dc3896e9be48b 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -29,6 +29,8 @@ export interface FindFileStructureResponse { count: number; cardinality: number; top_hits: Array<{ count: number; value: any }>; + max_value?: number; + min_value?: number; }; }; sample_start: string; @@ -42,7 +44,7 @@ export interface FindFileStructureResponse { delimiter: string; need_client_timezone: boolean; num_lines_analyzed: number; - column_names: string[]; + column_names?: string[]; explanation?: string[]; grok_pattern?: string; multiline_start_pattern?: string; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_field_label.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_field_label.tsx new file mode 100644 index 00000000000000..610b29c85a0627 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_field_label.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiText } from '@elastic/eui'; + +import { CombinedField } from './types'; + +export function CombinedFieldLabel({ combinedField }: { combinedField: CombinedField }) { + return {getCombinedFieldLabel(combinedField)}; +} + +function getCombinedFieldLabel(combinedField: CombinedField) { + return `${combinedField.fieldNames.join(combinedField.delimiter)} => ${ + combinedField.combinedFieldName + } (${combinedField.mappingType})`; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx new file mode 100644 index 00000000000000..fdfe10c2acf023 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { Component } from 'react'; + +import { + EuiFormRow, + EuiPopover, + EuiContextMenu, + EuiButtonEmpty, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; + +import { CombinedField } from './types'; +import { GeoPointForm } from './geo_point'; +import { CombinedFieldLabel } from './combined_field_label'; +import { + addCombinedFieldsToMappings, + addCombinedFieldsToPipeline, + getNameCollisionMsg, + removeCombinedFieldsFromMappings, + removeCombinedFieldsFromPipeline, +} from './utils'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; + +interface Props { + mappingsString: string; + pipelineString: string; + onMappingsStringChange(): void; + onPipelineStringChange(): void; + combinedFields: CombinedField[]; + onCombinedFieldsChange(combinedFields: CombinedField[]): void; + results: FindFileStructureResponse; + isDisabled: boolean; +} + +interface State { + isPopoverOpen: boolean; +} + +export class CombinedFieldsForm extends Component { + state: State = { + isPopoverOpen: false, + }; + + togglePopover = () => { + this.setState((prevState) => ({ + isPopoverOpen: !prevState.isPopoverOpen, + })); + }; + + closePopover = () => { + this.setState({ + isPopoverOpen: false, + }); + }; + + addCombinedField = (combinedField: CombinedField) => { + if (this.hasNameCollision(combinedField.combinedFieldName)) { + throw new Error(getNameCollisionMsg(combinedField.combinedFieldName)); + } + + const mappings = this.parseMappings(); + const pipeline = this.parsePipeline(); + + this.props.onMappingsStringChange( + // @ts-expect-error + JSON.stringify(addCombinedFieldsToMappings(mappings, [combinedField]), null, 2) + ); + this.props.onPipelineStringChange( + // @ts-expect-error + JSON.stringify(addCombinedFieldsToPipeline(pipeline, [combinedField]), null, 2) + ); + this.props.onCombinedFieldsChange([...this.props.combinedFields, combinedField]); + + this.closePopover(); + }; + + removeCombinedField = (index: number) => { + let mappings; + let pipeline; + try { + mappings = this.parseMappings(); + pipeline = this.parsePipeline(); + } catch (error) { + // how should remove error be surfaced? + return; + } + + const updatedCombinedFields = [...this.props.combinedFields]; + const removedCombinedFields = updatedCombinedFields.splice(index, 1); + + this.props.onMappingsStringChange( + // @ts-expect-error + JSON.stringify(removeCombinedFieldsFromMappings(mappings, removedCombinedFields), null, 2) + ); + this.props.onPipelineStringChange( + // @ts-expect-error + JSON.stringify(removeCombinedFieldsFromPipeline(pipeline, removedCombinedFields), null, 2) + ); + this.props.onCombinedFieldsChange(updatedCombinedFields); + }; + + parseMappings() { + try { + return JSON.parse(this.props.mappingsString); + } catch (error) { + throw new Error( + i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsForm.mappingsParseError', { + defaultMessage: 'Error parsing mappings: {error}', + values: { error: error.message }, + }) + ); + } + } + + parsePipeline() { + try { + return JSON.parse(this.props.pipelineString); + } catch (error) { + throw new Error( + i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsForm.pipelineParseError', { + defaultMessage: 'Error parsing pipeline: {error}', + values: { error: error.message }, + }) + ); + } + } + + hasNameCollision = (name: string) => { + if (this.props.results.column_names?.includes(name)) { + // collision with column name + return true; + } + + if ( + this.props.combinedFields.some((combinedField) => combinedField.combinedFieldName === name) + ) { + // collision with combined field name + return true; + } + + const mappings = this.parseMappings(); + return mappings.properties.hasOwnProperty(name); + }; + + render() { + const geoPointLabel = i18n.translate('xpack.ml.fileDatavisualizer.geoPointCombinedFieldLabel', { + defaultMessage: 'Add geo point field', + }); + const panels = [ + { + id: 0, + items: [ + { + name: geoPointLabel, + panel: 1, + }, + ], + }, + { + id: 1, + title: geoPointLabel, + content: ( + + ), + }, + ]; + return ( + +
+ {this.props.combinedFields.map((combinedField: CombinedField, idx: number) => ( + + + + + {!this.props.isDisabled && ( + + + + )} + + ))} + + + + } + isOpen={this.state.isPopoverOpen} + closePopover={this.closePopover} + anchorPosition="rightCenter" + > + + +
+ + ); + } +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_read_only_form.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_read_only_form.tsx new file mode 100644 index 00000000000000..c37e27e39a7aba --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_read_only_form.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +import { EuiFormRow } from '@elastic/eui'; + +import { CombinedField } from './types'; +import { CombinedFieldLabel } from './combined_field_label'; + +export function CombinedFieldsReadOnlyForm({ + combinedFields, +}: { + combinedFields: CombinedField[]; +}) { + return combinedFields.length ? ( + +
+ {combinedFields.map((combinedField: CombinedField, idx: number) => ( + + ))} +
+ + ) : null; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx new file mode 100644 index 00000000000000..831ae8de8081a9 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import debounce from 'lodash/debounce'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { ChangeEvent, Component, Fragment } from 'react'; + +import { + EuiFormRow, + EuiFieldText, + EuiTextAlign, + EuiSpacer, + EuiButton, + EuiSelect, + EuiSelectOption, + EuiFormErrorText, +} from '@elastic/eui'; + +import { CombinedField } from './types'; +import { + createGeoPointCombinedField, + isWithinLatRange, + isWithinLonRange, + getFieldNames, + getNameCollisionMsg, +} from './utils'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; + +interface Props { + addCombinedField: (combinedField: CombinedField) => void; + hasNameCollision: (name: string) => boolean; + results: FindFileStructureResponse; +} + +interface State { + latField: string; + lonField: string; + geoPointField: string; + geoPointFieldError: string; + latFields: EuiSelectOption[]; + lonFields: EuiSelectOption[]; + submitError: string; +} + +export class GeoPointForm extends Component { + constructor(props: Props) { + super(props); + + const latFields: EuiSelectOption[] = [{ value: '', text: '' }]; + const lonFields: EuiSelectOption[] = [{ value: '', text: '' }]; + getFieldNames(props.results).forEach((columnName: string) => { + if (isWithinLatRange(columnName, props.results.field_stats)) { + latFields.push({ value: columnName, text: columnName }); + } + if (isWithinLonRange(columnName, props.results.field_stats)) { + lonFields.push({ value: columnName, text: columnName }); + } + }); + + this.state = { + latField: '', + lonField: '', + geoPointField: '', + geoPointFieldError: '', + submitError: '', + latFields, + lonFields, + }; + } + + onLatFieldChange = (e: ChangeEvent) => { + this.setState({ latField: e.target.value }); + }; + + onLonFieldChange = (e: ChangeEvent) => { + this.setState({ lonField: e.target.value }); + }; + + onGeoPointFieldChange = (e: ChangeEvent) => { + const geoPointField = e.target.value; + this.setState({ geoPointField }); + this.hasNameCollision(geoPointField); + }; + + hasNameCollision = debounce((name: string) => { + try { + const geoPointFieldError = this.props.hasNameCollision(name) ? getNameCollisionMsg(name) : ''; + this.setState({ geoPointFieldError }); + } catch (error) { + this.setState({ submitError: error.message }); + } + }, 200); + + onSubmit = () => { + try { + this.props.addCombinedField( + createGeoPointCombinedField( + this.state.latField, + this.state.lonField, + this.state.geoPointField + ) + ); + this.setState({ submitError: '' }); + } catch (error) { + this.setState({ submitError: error.message }); + } + }; + + render() { + let error; + if (this.state.submitError) { + error = {this.state.submitError}; + } + return ( + + + + + + + + + + + + + + + + {error} + + + + + + + + ); + } +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/index.ts new file mode 100644 index 00000000000000..90b6bbab789f3b --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { + addCombinedFieldsToPipeline, + addCombinedFieldsToMappings, + getDefaultCombinedFields, +} from './utils'; + +export { CombinedFieldsReadOnlyForm } from './combined_fields_read_only_form'; +export { CombinedFieldsForm } from './combined_fields_form'; +export { CombinedField } from './types'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/types.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/types.ts new file mode 100644 index 00000000000000..1ec66f5c966610 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface CombinedField { + mappingType: string; + delimiter: string; + combinedFieldName: string; + fieldNames: string[]; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.test.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.test.ts new file mode 100644 index 00000000000000..17b39f9041ec07 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.test.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + addCombinedFieldsToMappings, + addCombinedFieldsToPipeline, + createGeoPointCombinedField, + isWithinLatRange, + isWithinLonRange, + removeCombinedFieldsFromMappings, + removeCombinedFieldsFromPipeline, +} from './utils'; + +const combinedFields = [createGeoPointCombinedField('lat', 'lon', 'location')]; + +test('addCombinedFieldsToMappings', () => { + const mappings = { + _meta: { + created_by: '', + }, + properties: { + lat: { + type: 'number', + }, + lon: { + type: 'number', + }, + }, + }; + expect(addCombinedFieldsToMappings(mappings, combinedFields)).toEqual({ + _meta: { + created_by: '', + }, + properties: { + lat: { + type: 'number', + }, + lon: { + type: 'number', + }, + location: { + type: 'geo_point', + }, + }, + }); +}); + +test('removeCombinedFieldsFromMappings', () => { + const mappings = { + _meta: { + created_by: '', + }, + properties: { + lat: { + type: 'number', + }, + lon: { + type: 'number', + }, + location: { + type: 'geo_point', + }, + }, + }; + expect(removeCombinedFieldsFromMappings(mappings, combinedFields)).toEqual({ + _meta: { + created_by: '', + }, + properties: { + lat: { + type: 'number', + }, + lon: { + type: 'number', + }, + }, + }); +}); + +test('addCombinedFieldsToPipeline', () => { + const pipeline = { + description: '', + processors: [ + { + set: { + field: 'anotherfield', + value: '{{value}}', + }, + }, + ], + }; + expect(addCombinedFieldsToPipeline(pipeline, combinedFields)).toEqual({ + description: '', + processors: [ + { + set: { + field: 'anotherfield', + value: '{{value}}', + }, + }, + { + set: { + field: 'location', + value: '{{lat}},{{lon}}', + }, + }, + ], + }); +}); + +test('removeCombinedFieldsFromPipeline', () => { + const pipeline = { + description: '', + processors: [ + { + set: { + field: 'anotherfield', + value: '{{value}}', + }, + }, + { + set: { + field: 'location', + value: '{{lat}},{{lon}}', + }, + }, + ], + }; + expect(removeCombinedFieldsFromPipeline(pipeline, combinedFields)).toEqual({ + description: '', + processors: [ + { + set: { + field: 'anotherfield', + value: '{{value}}', + }, + }, + ], + }); +}); + +test('isWithinLatRange', () => { + expect(isWithinLatRange('fieldAlpha', {})).toBe(false); + expect( + isWithinLatRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 1 }], + }, + }) + ).toBe(false); + expect( + isWithinLatRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 100 }], + max_value: 100, + min_value: 0, + }, + }) + ).toBe(false); + expect( + isWithinLatRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: -100 }], + max_value: 0, + min_value: -100, + }, + }) + ).toBe(false); + expect( + isWithinLatRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 0 }], + max_value: 0, + min_value: 0, + }, + }) + ).toBe(true); +}); + +test('isWithinLonRange', () => { + expect(isWithinLonRange('fieldAlpha', {})).toBe(false); + expect( + isWithinLonRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 1 }], + }, + }) + ).toBe(false); + expect( + isWithinLonRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 200 }], + max_value: 200, + min_value: 0, + }, + }) + ).toBe(false); + expect( + isWithinLonRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: -200 }], + max_value: 0, + min_value: -200, + }, + }) + ).toBe(false); + expect( + isWithinLonRange('fieldAlpha', { + fieldAlpha: { + count: 1, + cardinality: 1, + top_hits: [{ count: 1, value: 0 }], + max_value: 0, + min_value: 0, + }, + }) + ).toBe(true); +}); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts new file mode 100644 index 00000000000000..5e7de14f451c20 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import _ from 'lodash'; +import uuid from 'uuid/v4'; +import { CombinedField } from './types'; +import { + FindFileStructureResponse, + IngestPipeline, + Mappings, +} from '../../../../../../common/types/file_datavisualizer'; + +const COMMON_LAT_NAMES = ['latitude', 'lat']; +const COMMON_LON_NAMES = ['longitude', 'long', 'lon']; + +export function getDefaultCombinedFields(results: FindFileStructureResponse) { + const combinedFields: CombinedField[] = []; + const geoPointField = getGeoPointField(results); + if (geoPointField) { + combinedFields.push(geoPointField); + } + return combinedFields; +} + +export function addCombinedFieldsToMappings( + mappings: Mappings, + combinedFields: CombinedField[] +): Mappings { + const updatedMappings = { ...mappings }; + combinedFields.forEach((combinedField) => { + updatedMappings.properties[combinedField.combinedFieldName] = { + type: combinedField.mappingType, + }; + }); + return updatedMappings; +} + +export function removeCombinedFieldsFromMappings( + mappings: Mappings, + combinedFields: CombinedField[] +) { + const updatedMappings = { ...mappings }; + combinedFields.forEach((combinedField) => { + delete updatedMappings.properties[combinedField.combinedFieldName]; + }); + return updatedMappings; +} + +export function addCombinedFieldsToPipeline( + pipeline: IngestPipeline, + combinedFields: CombinedField[] +) { + const updatedPipeline = _.cloneDeep(pipeline); + combinedFields.forEach((combinedField) => { + updatedPipeline.processors.push({ + set: { + field: combinedField.combinedFieldName, + value: combinedField.fieldNames + .map((fieldName) => { + return `{{${fieldName}}}`; + }) + .join(combinedField.delimiter), + }, + }); + }); + return updatedPipeline; +} + +export function removeCombinedFieldsFromPipeline( + pipeline: IngestPipeline, + combinedFields: CombinedField[] +) { + return { + ...pipeline, + processors: pipeline.processors.filter((processor) => { + return 'set' in processor + ? !combinedFields.some((combinedField) => { + return processor.set.field === combinedField.combinedFieldName; + }) + : true; + }), + }; +} + +export function isWithinLatRange( + fieldName: string, + fieldStats: FindFileStructureResponse['field_stats'] +) { + return ( + fieldName in fieldStats && + 'max_value' in fieldStats[fieldName] && + fieldStats[fieldName]!.max_value! <= 90 && + 'min_value' in fieldStats[fieldName] && + fieldStats[fieldName]!.min_value! >= -90 + ); +} + +export function isWithinLonRange( + fieldName: string, + fieldStats: FindFileStructureResponse['field_stats'] +) { + return ( + fieldName in fieldStats && + 'max_value' in fieldStats[fieldName] && + fieldStats[fieldName]!.max_value! <= 180 && + 'min_value' in fieldStats[fieldName] && + fieldStats[fieldName]!.min_value! >= -180 + ); +} + +export function createGeoPointCombinedField( + latField: string, + lonField: string, + geoPointField: string +): CombinedField { + return { + mappingType: 'geo_point', + delimiter: ',', + combinedFieldName: geoPointField, + fieldNames: [latField, lonField], + }; +} + +export function getNameCollisionMsg(name: string) { + return i18n.translate('xpack.ml.fileDatavisualizer.nameCollisionMsg', { + defaultMessage: '"{name}" already exists, please provide a unique name', + values: { name }, + }); +} + +export function getFieldNames(results: FindFileStructureResponse): string[] { + return results.column_names !== undefined + ? results.column_names + : Object.keys(results.field_stats); +} + +function getGeoPointField(results: FindFileStructureResponse) { + const fieldNames = getFieldNames(results); + + const latField = fieldNames.find((columnName) => { + return ( + COMMON_LAT_NAMES.includes(columnName.toLowerCase()) && + isWithinLatRange(columnName, results.field_stats) + ); + }); + + const lonField = fieldNames.find((columnName) => { + return ( + COMMON_LON_NAMES.includes(columnName.toLowerCase()) && + isWithinLonRange(columnName, results.field_stats) + ); + }); + + if (!latField || !lonField) { + return null; + } + + const combinedFieldNames = [ + 'location', + 'point_location', + `${latField}_${lonField}`, + `location_${uuid()}`, + ]; + // Use first combinedFieldNames that does not have a naming collision + const geoPointField = combinedFieldNames.find((name) => { + return !fieldNames.includes(name); + }); + + return geoPointField ? createGeoPointCombinedField(latField, lonField, geoPointField) : null; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx index a79a7d36f32945..2b49746170f46c 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx @@ -17,7 +17,9 @@ import { EuiFlexItem, } from '@elastic/eui'; +import { CombinedField, CombinedFieldsForm } from '../combined_fields'; import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; const EDITOR_HEIGHT = '300px'; interface Props { @@ -36,6 +38,9 @@ interface Props { onPipelineStringChange(): void; indexNameError: string; indexPatternNameError: string; + combinedFields: CombinedField[]; + onCombinedFieldsChange(combinedFields: CombinedField[]): void; + results: FindFileStructureResponse; } export const AdvancedSettings: FC = ({ @@ -54,6 +59,9 @@ export const AdvancedSettings: FC = ({ onPipelineStringChange, indexNameError, indexPatternNameError, + combinedFields, + onCombinedFieldsChange, + results, }) => { return ( @@ -123,6 +131,17 @@ export const AdvancedSettings: FC = ({ /> + + = ({ @@ -46,6 +51,9 @@ export const ImportSettings: FC = ({ onPipelineStringChange, indexNameError, indexPatternNameError, + combinedFields, + onCombinedFieldsChange, + results, }) => { const tabs = [ { @@ -64,6 +72,7 @@ export const ImportSettings: FC = ({ createIndexPattern={createIndexPattern} onCreateIndexPatternChange={onCreateIndexPatternChange} indexNameError={indexNameError} + combinedFields={combinedFields} /> ), @@ -93,6 +102,9 @@ export const ImportSettings: FC = ({ onPipelineStringChange={onPipelineStringChange} indexNameError={indexNameError} indexPatternNameError={indexPatternNameError} + combinedFields={combinedFields} + onCombinedFieldsChange={onCombinedFieldsChange} + results={results} /> ), diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx index 1e716824729e33..f6cd5909cbb802 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx @@ -9,6 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import React, { FC } from 'react'; import { EuiFieldText, EuiFormRow, EuiCheckbox, EuiSpacer } from '@elastic/eui'; +import { CombinedField, CombinedFieldsReadOnlyForm } from '../combined_fields'; interface Props { index: string; @@ -17,6 +18,7 @@ interface Props { createIndexPattern: boolean; onCreateIndexPatternChange(): void; indexNameError: string; + combinedFields: CombinedField[]; } export const SimpleSettings: FC = ({ @@ -26,6 +28,7 @@ export const SimpleSettings: FC = ({ createIndexPattern, onCreateIndexPatternChange, indexNameError, + combinedFields, }) => { return ( @@ -75,6 +78,10 @@ export const SimpleSettings: FC = ({ onChange={onCreateIndexPatternChange} data-test-subj="mlFileDataVisCreateIndexPatternCheckbox" /> + + + + ); }; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js index 36b77a5a25e091..08b61a5fa4eed2 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js @@ -26,6 +26,11 @@ import { ImportProgress, IMPORT_STATUS } from '../import_progress'; import { ImportErrors } from '../import_errors'; import { ImportSummary } from '../import_summary'; import { ImportSettings } from '../import_settings'; +import { + addCombinedFieldsToPipeline, + addCombinedFieldsToMappings, + getDefaultCombinedFields, +} from '../combined_fields'; import { ExperimentalBadge } from '../experimental_badge'; import { getIndexPatternNames, loadIndexPatterns } from '../../../../util/index_utils'; import { ml } from '../../../../services/ml_api_service'; @@ -68,6 +73,7 @@ const DEFAULT_STATE = { timeFieldName: undefined, isFilebeatFlyoutVisible: false, checkingValidIndex: false, + combinedFields: [], }; export class ImportView extends Component { @@ -386,6 +392,10 @@ export class ImportView extends Component { }); }; + onCombinedFieldsChange = (combinedFields) => { + this.setState({ combinedFields }); + }; + setImportProgress = (progress) => { this.setState({ uploadProgress: progress, @@ -444,6 +454,7 @@ export class ImportView extends Component { timeFieldName, isFilebeatFlyoutVisible, checkingValidIndex, + combinedFields, } = this.state; const createPipeline = pipelineString !== ''; @@ -513,6 +524,9 @@ export class ImportView extends Component { onPipelineStringChange={this.onPipelineStringChange} indexNameError={indexNameError} indexPatternNameError={indexPatternNameError} + combinedFields={combinedFields} + onCombinedFieldsChange={this.onCombinedFieldsChange} + results={this.props.results} /> @@ -644,12 +658,22 @@ function getDefaultState(state, results) { ? JSON.stringify(DEFAULT_INDEX_SETTINGS, null, 2) : state.indexSettingsString; + const combinedFields = state.combinedFields.length + ? state.combinedFields + : getDefaultCombinedFields(results); + const mappingsString = - state.mappingsString === '' ? JSON.stringify(results.mappings, null, 2) : state.mappingsString; + state.mappingsString === '' + ? JSON.stringify(addCombinedFieldsToMappings(results.mappings, combinedFields), null, 2) + : state.mappingsString; const pipelineString = state.pipelineString === '' && results.ingest_pipeline !== undefined - ? JSON.stringify(results.ingest_pipeline, null, 2) + ? JSON.stringify( + addCombinedFieldsToPipeline(results.ingest_pipeline, combinedFields), + null, + 2 + ) : state.pipelineString; const timeFieldName = results.timestamp_field; @@ -660,6 +684,7 @@ function getDefaultState(state, results) { mappingsString, pipelineString, timeFieldName, + combinedFields, }; }