diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 6e5b5b16..fcc82042 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -4,10 +4,9 @@ */ import React, { useEffect, useState } from 'react'; -import { useFormikContext, getIn } from 'formik'; +import { useFormikContext } from 'formik'; import { EuiButton, - EuiCodeBlock, EuiFilePicker, EuiFlexGroup, EuiFlexItem, @@ -122,9 +121,15 @@ export function SourceData(props: SourceDataProps) { - - {getIn(values, 'ingest.docs')} - + {}} + editorHeight="25vh" + readOnly={true} + /> > diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 31a903f9..2199941f 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -5,11 +5,10 @@ import React, { useState } from 'react'; import { useFormikContext, getIn } from 'formik'; -import { isEmpty, get } from 'lodash'; -import jsonpath from 'jsonpath'; +import { isEmpty } from 'lodash'; import { EuiButton, - EuiCodeBlock, + EuiCodeEditor, EuiFlexGroup, EuiFlexItem, EuiModal, @@ -27,12 +26,16 @@ import { JSONPATH_ROOT_SELECTOR, ML_INFERENCE_DOCS_LINK, PROCESSOR_CONTEXT, - SimulateIngestPipelineDoc, SimulateIngestPipelineResponse, WorkflowConfig, WorkflowFormValues, } from '../../../../../common'; -import { formikToIngestPipeline, generateId } from '../../../../utils'; +import { + formikToIngestPipeline, + generateTransform, + prepareDocsForSimulate, + unwrapTransformedDocs, +} from '../../../../utils'; import { simulatePipeline, useAppDispatch } from '../../../../store'; import { getCore } from '../../../../services'; import { MapField } from '../input_fields'; @@ -58,8 +61,8 @@ export function InputTransformModal(props: InputTransformModalProps) { const [sourceInput, setSourceInput] = useState('[]'); const [transformedOutput, setTransformedOutput] = useState('[]'); - // parse out the values and determine if there are none/some/all valid jsonpaths - const mapValues = getIn(values, `ingest.enrich.${props.config.id}.inputMap`); + // get the current input map + const map = getIn(values, `ingest.enrich.${props.config.id}.inputMap`); return ( @@ -68,7 +71,7 @@ export function InputTransformModal(props: InputTransformModalProps) { {`Configure input`} - + <> @@ -78,10 +81,12 @@ export function InputTransformModal(props: InputTransformModalProps) { onClick={async () => { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, but not including, this processor const curIngestPipeline = formikToIngestPipeline( values, props.uiConfig, - props.config.id + props.config.id, + false ); // if there are preceding processors, we need to generate the ingest pipeline // up to this point and simulate, in order to get the latest transformed @@ -103,7 +108,7 @@ export function InputTransformModal(props: InputTransformModalProps) { }) .catch((error: any) => { getCore().notifications.toasts.addDanger( - `Failed to fetch input schema` + `Failed to fetch input data` ); }); } else { @@ -118,9 +123,22 @@ export function InputTransformModal(props: InputTransformModalProps) { Fetch - - {sourceInput} - + > @@ -148,72 +166,23 @@ export function InputTransformModal(props: InputTransformModalProps) { Expected output { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { - if ( - !isEmpty(mapValues) && - !isEmpty(JSON.parse(sourceInput)) - ) { - let output = {}; + if (!isEmpty(map) && !isEmpty(JSON.parse(sourceInput))) { let sampleSourceInput = {}; try { sampleSourceInput = JSON.parse(sourceInput)[0]; + const output = generateTransform( + sampleSourceInput, + map + ); + setTransformedOutput( + JSON.stringify(output, undefined, 2) + ); } catch {} - - mapValues.forEach( - (mapValue: { key: string; value: string }) => { - const path = mapValue.value; - try { - let transformedResult = undefined; - // ML inference processors will use standard dot notation or JSONPath depending on the input. - // We follow the same logic here to generate consistent results. - if ( - mapValue.value.startsWith( - JSONPATH_ROOT_SELECTOR - ) - ) { - // JSONPath transform - transformedResult = jsonpath.query( - sampleSourceInput, - path - ); - // Bracket notation not supported - throw an error - } else if ( - mapValue.value.includes(']') || - mapValue.value.includes(']') - ) { - throw new Error(); - // Standard dot notation - } else { - transformedResult = get( - sampleSourceInput, - path - ); - } - - output = { - ...output, - [mapValue.key]: transformedResult || '', - }; - - setTransformedOutput( - JSON.stringify(output, undefined, 2) - ); - } catch (e: any) { - console.error(e); - getCore().notifications.toasts.addDanger( - 'Error generating expected output. Ensure your inputs are valid JSONPath or dot notation syntax.', - e - ); - } - } - ); } - break; } // TODO: complete for search request / search response contexts @@ -223,9 +192,22 @@ export function InputTransformModal(props: InputTransformModalProps) { Generate - - {transformedOutput} - + > @@ -238,48 +220,3 @@ export function InputTransformModal(props: InputTransformModalProps) { ); } - -// docs are expected to be in a certain format to be passed to the simulate ingest pipeline API. -// for details, see https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest -function prepareDocsForSimulate( - docs: string, - indexName: string -): SimulateIngestPipelineDoc[] { - const preparedDocs = [] as SimulateIngestPipelineDoc[]; - const docObjs = JSON.parse(docs) as {}[]; - docObjs.forEach((doc) => { - preparedDocs.push({ - _index: indexName, - _id: generateId(), - _source: doc, - }); - }); - return preparedDocs; -} - -// docs are returned in a certain format from the simulate ingest pipeline API. We want -// to format them into a more readable string to display -function unwrapTransformedDocs( - simulatePipelineResponse: SimulateIngestPipelineResponse -) { - let errorDuringSimulate = undefined as string | undefined; - const transformedDocsSources = simulatePipelineResponse.docs.map( - (transformedDoc) => { - if (transformedDoc.error !== undefined) { - errorDuringSimulate = transformedDoc.error.reason || ''; - } else { - return transformedDoc.doc._source; - } - } - ); - - // there is an edge case where simulate may fail if there is some server-side or OpenSearch issue when - // running ingest (e.g., hitting rate limits on remote model) - // We pull out any returned error from a document and propagate it to the user. - if (errorDuringSimulate !== undefined) { - getCore().notifications.toasts.addDanger( - `Failed to simulate ingest on all documents: ${errorDuringSimulate}` - ); - } - return JSON.stringify(transformedDocsSources, undefined, 2); -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 01ab9894..adf7ca89 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -6,9 +6,9 @@ import React, { useState } from 'react'; import { getIn, useFormikContext } from 'formik'; import { - EuiButton, - EuiFilterButton, - EuiFilterGroup, + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, EuiSpacer, EuiText, } from '@elastic/eui'; @@ -64,11 +64,6 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { boolean >(false); - // advanced / simple transform state - const [isSimpleTransformSelected, setIsSimpleTransformSelected] = useState< - boolean - >(true); - return ( <> {isInputTransformModalOpen && ( @@ -84,11 +79,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { )} {isOutputTransformModalOpen && ( setIsOutputTransformModalOpen(false)} - onConfirm={() => { - console.log('saving transform output configuration...'); - setIsOutputTransformModalOpen(false); - }} /> )} - {`Configure data transformations (optional)`} + + + {`Configure input transformations (optional)`} + + + { + setIsInputTransformModalOpen(true); + }} + > + Preview + + + + {`Dot notation is used by default. To explicitly use JSONPath, please ensure to prepend with the root object selector "${JSONPATH_ROOT_SELECTOR}"`} - - setIsSimpleTransformSelected(true)} - > - Simple - - setIsSimpleTransformSelected(false)} - > - Advanced - - - - {isSimpleTransformSelected ? ( - <> - - - - > - ) : ( - <> - { - setIsInputTransformModalOpen(true); - }} - > - Configure input - - - - + + + + {`Configure output transformations (optional)`} + + + { setIsOutputTransformModalOpen(true); }} > - Configure output - - > - )} - + Preview + + + + + > )} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 3e8403ee..f7f6b21b 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -3,41 +3,220 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useState } from 'react'; +import { useFormikContext, getIn } from 'formik'; +import { cloneDeep, isEmpty, set } from 'lodash'; import { EuiButton, - EuiButtonEmpty, + EuiCodeEditor, + EuiFlexGroup, + EuiFlexItem, EuiModal, EuiModalBody, EuiModalFooter, EuiModalHeader, EuiModalHeaderTitle, + EuiSpacer, EuiText, } from '@elastic/eui'; +import { + IConfigField, + IProcessorConfig, + IngestPipelineConfig, + JSONPATH_ROOT_SELECTOR, + PROCESSOR_CONTEXT, + SimulateIngestPipelineResponse, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../common'; +import { + formikToIngestPipeline, + generateTransform, + prepareDocsForSimulate, + unwrapTransformedDocs, +} from '../../../../utils'; +import { simulatePipeline, useAppDispatch } from '../../../../store'; +import { getCore } from '../../../../services'; +import { MapField } from '../input_fields'; interface OutputTransformModalProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; + context: PROCESSOR_CONTEXT; + outputMapField: IConfigField; + outputMapFieldPath: string; onClose: () => void; - onConfirm: () => void; + onFormChange: () => void; } /** * A modal to configure advanced JSON-to-JSON transforms from a model's expected output */ export function OutputTransformModal(props: OutputTransformModalProps) { + const dispatch = useAppDispatch(); + const { values } = useFormikContext(); + + // source input / transformed output state + const [sourceInput, setSourceInput] = useState('[]'); + const [transformedOutput, setTransformedOutput] = useState('[]'); + + // get the current output map + const map = getIn(values, `ingest.enrich.${props.config.id}.outputMap`); + return ( - + - {`Configure output transform`} + {`Configure output`} - - TODO TODO TODO + + + + <> + Expected input + { + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, and including this processor. + // remove any currently-configured output map since we only want the transformation + // up to, and including, the input map transformations + const valuesWithoutOutputMapConfig = cloneDeep(values); + set( + valuesWithoutOutputMapConfig, + `ingest.enrich.${props.config.id}.outputMap`, + [] + ); + const curIngestPipeline = formikToIngestPipeline( + valuesWithoutOutputMapConfig, + props.uiConfig, + props.config.id, + true + ) as IngestPipelineConfig; + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.index.name + ); + await dispatch( + simulatePipeline({ + pipeline: curIngestPipeline, + docs: curDocs, + }) + ) + .unwrap() + .then((resp: SimulateIngestPipelineResponse) => { + setSourceInput(unwrapTransformedDocs(resp)); + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input data` + ); + }); + break; + } + // TODO: complete for search request / search response contexts + } + }} + > + Fetch + + + + > + + + <> + Define transform + + {`Dot notation is used by default. To explicitly use JSONPath, please ensure to prepend with the + root object selector "${JSONPATH_ROOT_SELECTOR}"`} + + + + > + + + <> + Expected output + { + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: { + if (!isEmpty(map) && !isEmpty(JSON.parse(sourceInput))) { + let sampleSourceInput = {}; + try { + sampleSourceInput = JSON.parse(sourceInput)[0]; + const output = generateTransform( + sampleSourceInput, + map + ); + setTransformedOutput( + JSON.stringify(output, undefined, 2) + ); + } catch {} + } + break; + } + // TODO: complete for search request / search response contexts + } + }} + > + Generate + + + + > + + - Cancel - - Save + + Close diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index d53f1937..c2c1211f 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -176,6 +176,7 @@ export function ProcessorsList(props: ProcessorsListProps) { { setPopover(!isPopoverOpen); }} diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx index a761f2fa..6d755c47 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx @@ -5,10 +5,9 @@ import React, { useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; -import { useFormikContext, getIn } from 'formik'; +import { useFormikContext } from 'formik'; import { EuiButton, - EuiCodeBlock, EuiFieldText, EuiFlexGroup, EuiFlexItem, @@ -25,10 +24,16 @@ import { } from '@elastic/eui'; import { WorkspaceFormValues } from '../../../../../common'; import { JsonField } from '../input_fields'; -import { AppState, catIndices, useAppDispatch } from '../../../../store'; +import { + AppState, + catIndices, + searchIndex, + useAppDispatch, +} from '../../../../store'; interface ConfigureSearchRequestProps { setQuery: (query: string) => void; + setQueryResponse: (queryResponse: string) => void; onFormChange: () => void; } @@ -142,9 +147,42 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { - - {getIn(values, 'search.request')} - + + + + { + // for this test query, we don't want to involve any configured search pipelines, if any exist + // see https://opensearch.org/docs/latest/search-plugins/search-pipelines/using-search-pipeline/#disabling-the-default-pipeline-for-a-request + dispatch( + searchIndex({ + index: indexName, + body: values.search.request, + searchPipeline: '_none', + }) + ) + .unwrap() + .then(async (resp) => { + const hits = resp.hits.hits; + props.setQueryResponse(JSON.stringify(hits, undefined, 2)); + }) + .catch((error: any) => { + props.setQueryResponse(''); + console.error('Error running query: ', error); + }); + }} + > + Test + > diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx index 8b4897f8..8a519d7a 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx @@ -14,6 +14,7 @@ interface SearchInputsProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; setQuery: (query: string) => void; + setQueryResponse: (queryResponse: string) => void; onFormChange: () => void; } @@ -26,6 +27,7 @@ export function SearchInputs(props: SearchInputsProps) { diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index 64970093..b8fadc09 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -532,6 +532,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} setQuery={props.setQuery} + setQueryResponse={props.setQueryResponse} onFormChange={props.onFormChange} /> ) : ( diff --git a/public/utils/form_to_pipeline_utils.ts b/public/utils/form_to_pipeline_utils.ts index 21f7f957..b36261c1 100644 --- a/public/utils/form_to_pipeline_utils.ts +++ b/public/utils/form_to_pipeline_utils.ts @@ -25,12 +25,14 @@ import { processorConfigsToTemplateProcessors } from './config_to_template_utils export function formikToIngestPipeline( values: WorkflowFormValues, existingConfig: WorkflowConfig, - curProcessorId: string + curProcessorId: string, + includeCurProcessor: boolean ): IngestPipelineConfig | SearchPipelineConfig | undefined { const uiConfig = formikToUiConfig(values, existingConfig); const precedingProcessors = getPrecedingProcessors( uiConfig.ingest.enrich.processors, - curProcessorId + curProcessorId, + includeCurProcessor ); if (!isEmpty(precedingProcessors)) { return { @@ -42,11 +44,15 @@ export function formikToIngestPipeline( function getPrecedingProcessors( allProcessors: IProcessorConfig[], - curProcessorId: string + curProcessorId: string, + includeCurProcessor: boolean ): IProcessorConfig[] { const precedingProcessors = [] as IProcessorConfig[]; allProcessors.some((processor) => { if (processor.id === curProcessorId) { + if (includeCurProcessor) { + precedingProcessors.push(processor); + } return true; } else { precedingProcessors.push(processor); diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 1987379f..4bd2b8b3 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -4,11 +4,17 @@ */ import yaml from 'js-yaml'; +import jsonpath from 'jsonpath'; +import { get } from 'lodash'; import { + JSONPATH_ROOT_SELECTOR, + SimulateIngestPipelineDoc, + SimulateIngestPipelineResponse, WORKFLOW_RESOURCE_TYPE, WORKFLOW_STEP_TYPE, Workflow, } from '../../common'; +import { getCore } from '../services'; // Append 16 random characters export function generateId(prefix?: string): string { @@ -103,3 +109,83 @@ export function isValidUiWorkflow(workflowObj: object | undefined): boolean { workflowObj?.ui_metadata?.type !== undefined ); } + +// Docs are expected to be in a certain format to be passed to the simulate ingest pipeline API. +// for details, see https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest +export function prepareDocsForSimulate( + docs: string, + indexName: string +): SimulateIngestPipelineDoc[] { + const preparedDocs = [] as SimulateIngestPipelineDoc[]; + const docObjs = JSON.parse(docs) as {}[]; + docObjs.forEach((doc) => { + preparedDocs.push({ + _index: indexName, + _id: generateId(), + _source: doc, + }); + }); + return preparedDocs; +} + +// Docs are returned in a certain format from the simulate ingest pipeline API. We want +// to format them into a more readable string to display +export function unwrapTransformedDocs( + simulatePipelineResponse: SimulateIngestPipelineResponse +) { + let errorDuringSimulate = undefined as string | undefined; + const transformedDocsSources = simulatePipelineResponse.docs.map( + (transformedDoc) => { + if (transformedDoc.error !== undefined) { + errorDuringSimulate = transformedDoc.error.reason || ''; + } else { + return transformedDoc.doc._source; + } + } + ); + + // there is an edge case where simulate may fail if there is some server-side or OpenSearch issue when + // running ingest (e.g., hitting rate limits on remote model) + // We pull out any returned error from a document and propagate it to the user. + if (errorDuringSimulate !== undefined) { + getCore().notifications.toasts.addDanger( + `Failed to simulate ingest on all documents: ${errorDuringSimulate}` + ); + } + return JSON.stringify(transformedDocsSources, undefined, 2); +} + +// ML inference processors will use standard dot notation or JSONPath depending on the input. +// We follow the same logic here to generate consistent results. +export function generateTransform( + input: {}, + map: { key: string; value: string }[] +): {} { + let output = {}; + map.forEach((mapEntry) => { + const path = mapEntry.value; + try { + let transformedResult = undefined; + if (mapEntry.value.startsWith(JSONPATH_ROOT_SELECTOR)) { + // JSONPath transform + transformedResult = jsonpath.query(input, path); + // Non-JSONPath bracket notation not supported - throw an error + } else if (mapEntry.value.includes('[') || mapEntry.value.includes(']')) { + throw new Error(); + // Standard dot notation + } else { + transformedResult = get(input, path); + } + output = { + ...output, + [mapEntry.key]: transformedResult || '', + }; + } catch (e: any) { + getCore().notifications.toasts.addDanger( + 'Error generating expected output. Ensure your transforms are valid JSONPath or dot notation syntax.', + e + ); + } + }); + return output; +}
{`Configure input`}
{`Configure output transform`}
{`Configure output`}