[ML] add geo point combined field to CSV import (elastic#77117)

* [ML] add geo point combined field to CSV import

* remove some geo_point specific logic

* Account for properties layer in find_file_structure mappings

* improve checking of name collision to include combined fields and mappings

* add delete button

* fix function name

* fill in unknowns with defined types

* tslint changes

* get tslint passing

* show readonly combined fields in simple tab

* handle column_names being undefined

* add unit tests for modifying mappings and pipeline

* review feedback

* do not change combinedFields on reset

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ml/common/types/file_datavisualizer.ts

@@ -29,6 +29,8 @@ export interface FindFileStructureResponse {
  count: number;
  cardinality: number;
  top_hits: Array<{ count: number; value: any }>;
+ max_value?: number;
+ min_value?: number;
  };
  };
  sample_start: string;
@@ -42,7 +44,7 @@ export interface FindFileStructureResponse {
  delimiter: string;
  need_client_timezone: boolean;
  num_lines_analyzed: number;
- column_names: string[];
+ column_names?: string[];
  explanation?: string[];
  grok_pattern?: string;
  multiline_start_pattern?: string;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_field_label.tsx

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiText } from '@elastic/eui';
+
+import { CombinedField } from './types';
+
+export function CombinedFieldLabel({ combinedField }: { combinedField: CombinedField }) {
+ return <EuiText size="s">{getCombinedFieldLabel(combinedField)}</EuiText>;
+}
+
+function getCombinedFieldLabel(combinedField: CombinedField) {
+ return `${combinedField.fieldNames.join(combinedField.delimiter)} => ${
+ combinedField.combinedFieldName
+ } (${combinedField.mappingType})`;
+}

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx

@@ -0,0 +1,237 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import React, { Component } from 'react';
+
+import {
+ EuiFormRow,
+ EuiPopover,
+ EuiContextMenu,
+ EuiButtonEmpty,
+ EuiButtonIcon,
+ EuiFlexGroup,
+ EuiFlexItem,
+} from '@elastic/eui';
+
+import { CombinedField } from './types';
+import { GeoPointForm } from './geo_point';
+import { CombinedFieldLabel } from './combined_field_label';
+import {
+ addCombinedFieldsToMappings,
+ addCombinedFieldsToPipeline,
+ getNameCollisionMsg,
+ removeCombinedFieldsFromMappings,
+ removeCombinedFieldsFromPipeline,
+} from './utils';
+import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+
+interface Props {
+ mappingsString: string;
+ pipelineString: string;
+ onMappingsStringChange(): void;
+ onPipelineStringChange(): void;
+ combinedFields: CombinedField[];
+ onCombinedFieldsChange(combinedFields: CombinedField[]): void;
+ results: FindFileStructureResponse;
+ isDisabled: boolean;
+}
+
+interface State {
+ isPopoverOpen: boolean;
+}
+
+export class CombinedFieldsForm extends Component<Props, State> {
+ state: State = {
+ isPopoverOpen: false,
+ };
+
+ togglePopover = () => {
+ this.setState((prevState) => ({
+ isPopoverOpen: !prevState.isPopoverOpen,
+ }));
+ };
+
+ closePopover = () => {
+ this.setState({
+ isPopoverOpen: false,
+ });
+ };
+
+ addCombinedField = (combinedField: CombinedField) => {
+ if (this.hasNameCollision(combinedField.combinedFieldName)) {
+ throw new Error(getNameCollisionMsg(combinedField.combinedFieldName));
+ }
+
+ const mappings = this.parseMappings();
+ const pipeline = this.parsePipeline();
+
+ this.props.onMappingsStringChange(
+ // @ts-expect-error
+ JSON.stringify(addCombinedFieldsToMappings(mappings, [combinedField]), null, 2)
+ );
+ this.props.onPipelineStringChange(
+ // @ts-expect-error
+ JSON.stringify(addCombinedFieldsToPipeline(pipeline, [combinedField]), null, 2)
+ );
+ this.props.onCombinedFieldsChange([...this.props.combinedFields, combinedField]);
+
+ this.closePopover();
+ };
+
+ removeCombinedField = (index: number) => {
+ let mappings;
+ let pipeline;
+ try {
+ mappings = this.parseMappings();
+ pipeline = this.parsePipeline();
+ } catch (error) {
+ // how should remove error be surfaced?
+ return;
+ }
+
+ const updatedCombinedFields = [...this.props.combinedFields];
+ const removedCombinedFields = updatedCombinedFields.splice(index, 1);
+
+ this.props.onMappingsStringChange(
+ // @ts-expect-error
+ JSON.stringify(removeCombinedFieldsFromMappings(mappings, removedCombinedFields), null, 2)
+ );
+ this.props.onPipelineStringChange(
+ // @ts-expect-error
+ JSON.stringify(removeCombinedFieldsFromPipeline(pipeline, removedCombinedFields), null, 2)
+ );
+ this.props.onCombinedFieldsChange(updatedCombinedFields);
+ };
+
+ parseMappings() {
+ try {
+ return JSON.parse(this.props.mappingsString);
+ } catch (error) {
+ throw new Error(
+ i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsForm.mappingsParseError', {
+ defaultMessage: 'Error parsing mappings: {error}',
+ values: { error: error.message },
+ })
+ );
+ }
+ }
+
+ parsePipeline() {
+ try {
+ return JSON.parse(this.props.pipelineString);
+ } catch (error) {
+ throw new Error(
+ i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsForm.pipelineParseError', {
+ defaultMessage: 'Error parsing pipeline: {error}',
+ values: { error: error.message },
+ })
+ );
+ }
+ }
+
+ hasNameCollision = (name: string) => {
+ if (this.props.results.column_names?.includes(name)) {
+ // collision with column name
+ return true;
+ }
+
+ if (
+ this.props.combinedFields.some((combinedField) => combinedField.combinedFieldName === name)
+ ) {
+ // collision with combined field name
+ return true;
+ }
+
+ const mappings = this.parseMappings();
+ return mappings.properties.hasOwnProperty(name);
+ };
+
+ render() {
+ const geoPointLabel = i18n.translate('xpack.ml.fileDatavisualizer.geoPointCombinedFieldLabel', {
+ defaultMessage: 'Add geo point field',
+ });
+ const panels = [
+ {
+ id: 0,
+ items: [
+ {
+ name: geoPointLabel,
+ panel: 1,
+ },
+ ],
+ },
+ {
+ id: 1,
+ title: geoPointLabel,
+ content: (
+ <GeoPointForm
+ addCombinedField={this.addCombinedField}
+ hasNameCollision={this.hasNameCollision}
+ results={this.props.results}
+ />
+ ),
+ },
+ ];
+ return (
+ <EuiFormRow
+ label={i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsLabel', {
+ defaultMessage: 'Combined fields',
+ })}
+ >
+ <div>
+ {this.props.combinedFields.map((combinedField: CombinedField, idx: number) => (
+ <EuiFlexGroup key={idx} gutterSize="s">
+ <EuiFlexItem>
+ <CombinedFieldLabel combinedField={combinedField} />
+ </EuiFlexItem>
+ {!this.props.isDisabled && (
+ <EuiFlexItem grow={false}>
+ <EuiButtonIcon
+ iconType="trash"
+ color="danger"
+ onClick={this.removeCombinedField.bind(null, idx)}
+ title={i18n.translate('xpack.ml.fileDatavisualizer.removeCombinedFieldsLabel', {
+ defaultMessage: 'Remove combined field',
+ })}
+ aria-label={i18n.translate(
+ 'xpack.ml.fileDatavisualizer.removeCombinedFieldsLabel',
+ {
+ defaultMessage: 'Remove combined field',
+ }
+ )}
+ />
+ </EuiFlexItem>
+ )}
+ </EuiFlexGroup>
+ ))}
+ <EuiPopover
+ id="combineFieldsPopover"
+ button={
+ <EuiButtonEmpty
+ onClick={this.togglePopover}
+ size="xs"
+ iconType="plusInCircleFilled"
+ isDisabled={this.props.isDisabled}
+ >
+ <FormattedMessage
+ id="xpack.ml.fileDatavisualizer.addCombinedFieldsLabel"
+ defaultMessage="Add combined field"
+ />
+ </EuiButtonEmpty>
+ }
+ isOpen={this.state.isPopoverOpen}
+ closePopover={this.closePopover}
+ anchorPosition="rightCenter"
+ >
+ <EuiContextMenu initialPanelId={0} panels={panels} />
+ </EuiPopover>
+ </div>
+ </EuiFormRow>
+ );
+ }
+}

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_read_only_form.tsx

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+
+import { EuiFormRow } from '@elastic/eui';
+
+import { CombinedField } from './types';
+import { CombinedFieldLabel } from './combined_field_label';
+
+export function CombinedFieldsReadOnlyForm({
+ combinedFields,
+}: {
+ combinedFields: CombinedField[];
+}) {
+ return combinedFields.length ? (
+ <EuiFormRow
+ label={i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsReadOnlyLabel', {
+ defaultMessage: 'Combined fields',
+ })}
+ helpText={i18n.translate('xpack.ml.fileDatavisualizer.combinedFieldsReadOnlyHelpTextLabel', {
+ defaultMessage: 'Edit combined fields in advanced tab',
+ })}
+ >
+ <div>
+ {combinedFields.map((combinedField: CombinedField, idx: number) => (
+ <CombinedFieldLabel key={idx} combinedField={combinedField} />
+ ))}
+ </div>
+ </EuiFormRow>
+ ) : null;
+}