Skip to content

Commit

Permalink
attach existing ml inference pipeline
Browse files Browse the repository at this point in the history
Added the ability to choose an existing ml inference pipeline and attach
it to the index. This will re-use the existing pipeline instead of
creating a new one.
  • Loading branch information
TattdCodeMonkey committed Oct 26, 2022
1 parent e0b5bdb commit f8e18e3
Show file tree
Hide file tree
Showing 16 changed files with 658 additions and 69 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
* 2.0.
*/

import { IngestSetProcessor, MlTrainedModelConfig } from '@elastic/elasticsearch/lib/api/types';
import {
IngestPipeline,
IngestSetProcessor,
MlTrainedModelConfig,
} from '@elastic/elasticsearch/lib/api/types';

import { MlInferencePipeline } from '../types/pipelines';
import { MlInferencePipeline, CreateMlInferencePipelineParameters } from '../types/pipelines';

// Getting an error importing this from @kbn/ml-plugin/common/constants/data_frame_analytics'
// So defining it locally for now with a test to make sure it matches.
Expand Down Expand Up @@ -151,3 +155,25 @@ export const formatPipelineName = (rawName: string) =>
.trim()
.replace(/\s+/g, '_') // Convert whitespaces to underscores
.toLowerCase();

export const parseMlInferenceParametersFromPipeline = (
name: string,
pipeline: IngestPipeline
): CreateMlInferencePipelineParameters | null => {
const processor = pipeline?.processors?.find((proc) => proc.inference !== undefined);
if (!processor || processor?.inference === undefined) {
return null;
}
const { inference: inferenceProcessor } = processor;
const sourceFields = Object.keys(inferenceProcessor.field_map ?? {});
const sourceField = sourceFields.length === 1 ? sourceFields[0] : null;
if (!sourceField) {
return null;
}
return {
destination_field: inferenceProcessor.target_field.replace('ml.inference.', ''),
model_id: inferenceProcessor.model_id,
pipeline_name: name,
source_field: sourceField,
};
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { mockHttpValues } from '../../../__mocks__/kea_logic';

import {
attachMlInferencePipeline,
AttachMlInferencePipelineApiLogicArgs,
AttachMlInferencePipelineResponse,
} from './attach_ml_inference_pipeline';

describe('AttachMlInferencePipelineApiLogic', () => {
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
});
describe('createMlInferencePipeline', () => {
it('calls the api', async () => {
const response: Promise<AttachMlInferencePipelineResponse> = Promise.resolve({
addedToParentPipeline: true,
created: false,
id: 'unit-test',
});
http.post.mockReturnValue(response);

const args: AttachMlInferencePipelineApiLogicArgs = {
indexName: 'unit-test-index',
pipelineName: 'unit-test',
};
const result = await attachMlInferencePipeline(args);
expect(http.post).toHaveBeenCalledWith(
'/internal/enterprise_search/indices/unit-test-index/ml_inference/pipeline_processors/attach',
{
body: '{"pipeline_name":"unit-test"}',
}
);
expect(result).toEqual({
addedToParentPipeline: true,
created: false,
id: args.pipelineName,
});
});
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { AttachMlInferencePipelineResponse } from '../../../../../common/types/pipelines';

import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../shared/http';

export interface AttachMlInferencePipelineApiLogicArgs {
indexName: string;
pipelineName: string;
}

export type { AttachMlInferencePipelineResponse };

export const attachMlInferencePipeline = async (
args: AttachMlInferencePipelineApiLogicArgs
): Promise<AttachMlInferencePipelineResponse> => {
const route = `/internal/enterprise_search/indices/${args.indexName}/ml_inference/pipeline_processors/attach`;
const params = {
pipeline_name: args.pipelineName,
};

return await HttpLogic.values.http.post<AttachMlInferencePipelineResponse>(route, {
body: JSON.stringify(params),
});
};

export const AttachMlInferencePipelineApiLogic = createApiLogic(
['attach_ml_inference_pipeline_api_logic'],
attachMlInferencePipeline
);
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,18 @@ import { InferencePipeline } from '../../../../../common/types/pipelines';
import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../shared/http';

export const fetchMlInferencePipelineProcessors = async ({ indexName }: { indexName: string }) => {
export interface FetchMlInferencePipelineProcessorsApiLogicArgs {
indexName: string;
}

export type FetchMlInferencePipelineProcessorsResponse = InferencePipeline[];

export const fetchMlInferencePipelineProcessors = async ({
indexName,
}: FetchMlInferencePipelineProcessorsApiLogicArgs) => {
const route = `/internal/enterprise_search/indices/${indexName}/ml_inference/pipeline_processors`;

return await HttpLogic.values.http.get<InferencePipeline[]>(route);
return await HttpLogic.values.http.get<FetchMlInferencePipelineProcessorsResponse>(route);
};

export const FetchMlInferencePipelineProcessorsApiLogic = createApiLogic(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { MlInferencePipeline } from '../../../../../common/types/pipelines';
import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../shared/http';

export type FetchMlInferencePipelinesArgs = undefined;
export type FetchMlInferencePipelinesResponse = Record<string, MlInferencePipeline | undefined>;

export const fetchMlInferencePipelines = async () => {
const route = '/internal/enterprise_search/pipelines/ml_inference';

return await HttpLogic.values.http.get<FetchMlInferencePipelinesResponse>(route);
};

export const FetchMlInferencePipelinesApiLogic = createApiLogic(
['fetch_ml_inference_pipelines_api_logic'],
fetchMlInferencePipelines
);
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ const AddProcessorContent: React.FC<AddMLInferencePipelineModalProps> = ({ onClo
</EuiModalBody>
);
}
if (supportedMLModels === undefined || supportedMLModels?.length === 0) {
if (supportedMLModels.length === 0) {
return <NoModelsPanel />;
}
return (
Expand Down Expand Up @@ -188,8 +188,10 @@ const ModalFooter: React.FC<AddMLInferencePipelineModalProps & { ingestionMethod
onClose,
}) => {
const { addInferencePipelineModal: modal, isPipelineDataValid } = useValues(MLInferenceLogic);
const { createPipeline, setAddInferencePipelineStep } = useActions(MLInferenceLogic);
const { attachPipeline, createPipeline, setAddInferencePipelineStep } =
useActions(MLInferenceLogic);

const attachExistingPipeline = Boolean(modal.configuration.existingPipeline);
let nextStep: AddInferencePipelineSteps | undefined;
let previousStep: AddInferencePipelineSteps | undefined;
switch (modal.step) {
Expand Down Expand Up @@ -239,6 +241,21 @@ const ModalFooter: React.FC<AddMLInferencePipelineModalProps & { ingestionMethod
>
{CONTINUE_BUTTON_LABEL}
</EuiButton>
) : attachExistingPipeline ? (
<EuiButton
color="primary"
data-telemetry-id={`entSearchContent-${ingestionMethod}-pipelines-addMlInference-attach`}
disabled={!isPipelineDataValid}
fill
onClick={attachPipeline}
>
{i18n.translate(
'xpack.enterpriseSearch.content.indices.transforms.addInferencePipelineModal.footer.attach',
{
defaultMessage: 'Attach',
}
)}
</EuiButton>
) : (
<EuiButton
color="success"
Expand Down
Loading

0 comments on commit f8e18e3

Please sign in to comment.