diff --git a/x-pack/plugins/ml/common/types/trained_models.ts b/x-pack/plugins/ml/common/types/trained_models.ts index 8798ed5ccb5a2..2dea2eecd5e5a 100644 --- a/x-pack/plugins/ml/common/types/trained_models.ts +++ b/x-pack/plugins/ml/common/types/trained_models.ts @@ -66,6 +66,7 @@ export type PutTrainedModelConfig = { model_aliases?: string[]; } & Record; tags?: string[]; + model_type?: TrainedModelType; inference_config?: Record; input: { field_names: string[] }; } & XOR< diff --git a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts index ca360130b89f9..4346ad0815e1c 100644 --- a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts +++ b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts @@ -6,14 +6,24 @@ */ import { FtrProviderContext } from '../../../../ftr_provider_context'; +import { SUPPORTED_TRAINED_MODELS } from '../../../../services/ml/api'; export default function ({ getService }: FtrProviderContext) { const ml = getService('ml'); + const trainedModels = Object.values(SUPPORTED_TRAINED_MODELS).map((model) => ({ + ...model, + id: model.name, + })); + describe('trained models', function () { before(async () => { - await ml.trainedModels.createTestTrainedModels('classification', 15, true); - await ml.trainedModels.createTestTrainedModels('regression', 15); + for (const model of trainedModels) { + await ml.api.importTrainedModel(model.id, model.name); + } + + await ml.api.createTestTrainedModels('classification', 15, true); + await ml.api.createTestTrainedModels('regression', 15); }); after(async () => { @@ -56,7 +66,7 @@ export default function ({ getService }: FtrProviderContext) { 'should display the stats bar with the total number of models' ); // +1 because of the built-in model - await ml.trainedModels.assertStats(31); + await ml.trainedModels.assertStats(37); await ml.testExecution.logTestStep('should display the table'); await ml.trainedModels.assertTableExists(); @@ -81,6 +91,16 @@ export default function ({ getService }: FtrProviderContext) { await ml.trainedModelsTable.assertPipelinesTabContent(false); }); + for (const model of trainedModels) { + it(`renders expanded row content correctly for imported tiny model ${model.id} without pipelines`, async () => { + await ml.trainedModelsTable.ensureRowIsExpanded(model.id); + await ml.trainedModelsTable.assertDetailsTabContent(); + await ml.trainedModelsTable.assertInferenceConfigTabContent(); + await ml.trainedModelsTable.assertStatsTabContent(); + await ml.trainedModelsTable.assertPipelinesTabContent(false); + }); + } + it('displays the built-in model and no actions are enabled', async () => { await ml.testExecution.logTestStep('should display the model in the table'); await ml.trainedModelsTable.filterWithSearchString(builtInModelData.modelId, 1); diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index 55d5a978dae82..d64f238a66bdd 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -9,7 +9,6 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import expect from '@kbn/expect'; import { ProvidedType } from '@kbn/test'; import fs from 'fs'; -import path from 'path'; import { Calendar } from '@kbn/ml-plugin/server/models/calendar'; import { Annotation } from '@kbn/ml-plugin/common/types/annotations'; import { DataFrameAnalyticsConfig } from '@kbn/ml-plugin/public/application/data_frame_analytics/common'; @@ -30,6 +29,45 @@ export type MlApi = ProvidedType; type ModelType = 'regression' | 'classification'; +export const SUPPORTED_TRAINED_MODELS = { + TINY_FILL_MASK: { + name: 'pt_tiny_fill_mask', + description: 'Tiny/Dummy PyTorch model (fill_mask)', + modelTypes: ['pytorch', 'fill_mask'], + }, + TINY_NER: { + name: 'pt_tiny_ner', + description: 'Tiny/Dummy PyTorch model (ner)', + modelTypes: ['pytorch', 'ner'], + }, + TINY_PASS_THROUGH: { + name: 'pt_tiny_pass_through', + description: 'Tiny/Dummy PyTorch model (pass_through)', + modelTypes: ['pytorch', 'pass_through'], + }, + TINY_TEXT_CLASSIFICATION: { + name: 'pt_tiny_text_classification', + description: 'Tiny/Dummy PyTorch model (text_classification)', + modelTypes: ['pytorch', 'text_classification'], + }, + TINY_TEXT_EMBEDDING: { + name: 'pt_tiny_text_embedding', + description: 'Tiny/Dummy PyTorch model (text_embedding)', + modelTypes: ['pytorch', 'text_embedding'], + }, + TINY_ZERO_SHOT: { + name: 'pt_tiny_zero_shot', + description: 'Tiny/Dummy PyTorch model (zero_shot)', + modelTypes: ['pytorch', 'zero_shot'], + }, +} as const; +export type SupportedTrainedModelNamesType = + typeof SUPPORTED_TRAINED_MODELS[keyof typeof SUPPORTED_TRAINED_MODELS]['name']; + +export interface TrainedModelVocabulary { + vocabulary: string[]; +} + export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { const es = getService('es'); const log = getService('log'); @@ -1135,6 +1173,38 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { return model; }, + async createTrainedModelVocabularyES(modelId: string, body: TrainedModelVocabulary) { + log.debug(`Creating vocabulary for trained model "${modelId}"`); + const { body: responseBody, status } = await esSupertest + .put(`/_ml/trained_models/${modelId}/vocabulary`) + .send(body); + this.assertResponseStatusCode(200, status, responseBody); + + log.debug('> Trained model vocabulary created'); + }, + + /** + * For the purpose of the functional tests where we only deal with very + * small models, we assume that the model definition can be uploaded as + * one part. + */ + async uploadTrainedModelDefinitionES(modelId: string, modelDefinitionPath: string) { + log.debug(`Uploading definition for trained model "${modelId}"`); + + const body = { + total_definition_length: fs.statSync(modelDefinitionPath).size, + definition: fs.readFileSync(modelDefinitionPath).toString('base64'), + total_parts: 1, + }; + + const { body: responseBody, status } = await esSupertest + .put(`/_ml/trained_models/${modelId}/definition/0`) + .send(body); + this.assertResponseStatusCode(200, status, responseBody); + + log.debug('> Trained model definition uploaded'); + }, + async deleteTrainedModelES(modelId: string) { log.debug(`Creating trained model with id "${modelId}"`); const { body: model, status } = await esSupertest.delete(`/_ml/trained_models/${modelId}`); @@ -1149,24 +1219,9 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { count: number = 10, withIngestPipelines = false ) { - const compressedDefinition = this.getCompressedModelDefinition(modelType); + const modelIds = new Array(count).fill(null).map((_v, i) => `dfa_${modelType}_model_n_${i}`); - const modelIds = new Array(count).fill(null).map((v, i) => `dfa_${modelType}_model_n_${i}`); - - const models = modelIds.map((id) => { - return { - model_id: id, - body: { - compressed_definition: compressedDefinition, - inference_config: { - [modelType]: {}, - }, - input: { - field_names: ['common_field'], - }, - } as PutTrainedModelConfig, - }; - }); + const models = modelIds.map((id) => this.createTestTrainedModelConfig(id, modelType)); for (const model of models) { await this.createTrainedModel(model.model_id, model.body); @@ -1178,7 +1233,7 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { return modelIds; }, - async createTestTrainedModelConfig(modelId: string, modelType: ModelType) { + createTestTrainedModelConfig(modelId: string, modelType: ModelType) { const compressedDefinition = this.getCompressedModelDefinition(modelType); return { @@ -1201,16 +1256,44 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { */ getCompressedModelDefinition(modelType: ModelType) { return fs.readFileSync( - path.resolve( - __dirname, - 'resources', - 'trained_model_definitions', - `minimum_valid_config_${modelType}.json.gz.b64` + require.resolve( + `./resources/trained_model_definitions/minimum_valid_config_${modelType}.json.gz.b64` ), 'utf-8' ); }, + getTrainedModelConfig(modelName: SupportedTrainedModelNamesType) { + const configFileContent = fs.readFileSync( + require.resolve(`./resources/trained_model_definitions/${modelName}/config.json`), + 'utf-8' + ); + return JSON.parse(configFileContent) as PutTrainedModelConfig; + }, + + getTrainedModelVocabulary(modelName: SupportedTrainedModelNamesType) { + const vocabularyFileContent = fs.readFileSync( + require.resolve(`./resources/trained_model_definitions/${modelName}/vocabulary.json`), + 'utf-8' + ); + return JSON.parse(vocabularyFileContent) as TrainedModelVocabulary; + }, + + getTrainedModelDefinitionPath(modelName: SupportedTrainedModelNamesType) { + return require.resolve( + `./resources/trained_model_definitions/${modelName}/traced_pytorch_model.pt` + ); + }, + + async importTrainedModel(modelId: string, modelName: SupportedTrainedModelNamesType) { + await this.createTrainedModel(modelId, this.getTrainedModelConfig(modelName)); + await this.createTrainedModelVocabularyES(modelId, this.getTrainedModelVocabulary(modelName)); + await this.uploadTrainedModelDefinitionES( + modelId, + this.getTrainedModelDefinitionPath(modelName) + ); + }, + async createModelAlias(modelId: string, modelAlias: string) { log.debug(`Creating alias for model "${modelId}"`); const { body, status } = await esSupertest.put( diff --git a/x-pack/test/functional/services/ml/index.ts b/x-pack/test/functional/services/ml/index.ts index ae7cb38e1c695..561bf4f6026b5 100644 --- a/x-pack/test/functional/services/ml/index.ts +++ b/x-pack/test/functional/services/ml/index.ts @@ -124,7 +124,7 @@ export function MachineLearningProvider(context: FtrProviderContext) { const testResources = MachineLearningTestResourcesProvider(context, api); const alerting = MachineLearningAlertingProvider(context, api, commonUI); const swimLane = SwimLaneProvider(context); - const trainedModels = TrainedModelsProvider(context, api, commonUI); + const trainedModels = TrainedModelsProvider(context, commonUI); const trainedModelsTable = TrainedModelsTableProvider(context, commonUI); const mlNodesPanel = MlNodesPanelProvider(context); diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/config.json new file mode 100644 index 0000000000000..cf00e1ade82ed --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/config.json @@ -0,0 +1,7 @@ +{ + "description": "Tiny/Dummy PyTorch model (fill_mask)", + "model_type": "pytorch", + "inference_config": { + "fill_mask": {} + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/traced_pytorch_model.pt new file mode 100644 index 0000000000000..6e7648e0ca77a Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/vocabulary.json new file mode 100644 index 0000000000000..9dc30191a0ca4 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_fill_mask/vocabulary.json @@ -0,0 +1,16 @@ +{ + "vocabulary": [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "Hello", + "world", + "car", + "bike", + "bee", + "bird", + "and" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/config.json new file mode 100644 index 0000000000000..2d50493dbc938 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/config.json @@ -0,0 +1,7 @@ +{ + "description": "Tiny/Dummy PyTorch model (ner)", + "model_type": "pytorch", + "inference_config": { + "ner": {} + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/traced_pytorch_model.pt new file mode 100644 index 0000000000000..35e36941c2e79 Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/vocabulary.json new file mode 100644 index 0000000000000..4f4511073f4fc --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_ner/vocabulary.json @@ -0,0 +1,21 @@ +{ + "vocabulary": [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "Hello", + "world", + "car", + "bike", + "bee", + "bird", + "and", + "my", + "name", + "is", + "I'm", + "Spartacus" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/config.json new file mode 100644 index 0000000000000..399aff4e2ca87 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/config.json @@ -0,0 +1,13 @@ +{ + "description": "Tiny/Dummy PyTorch model (pass_through)", + "model_type": "pytorch", + "inference_config": { + "pass_through": { + "tokenization": { + "bert": { + "with_special_tokens": false + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/traced_pytorch_model.pt new file mode 100644 index 0000000000000..0eecbb1b3f930 Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/vocabulary.json new file mode 100644 index 0000000000000..c8a39ea17a68c --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_pass_through/vocabulary.json @@ -0,0 +1,5 @@ +{ + "vocabulary": [ + "[UNK]", "[PAD]", "there", "is", "no", "spoon" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/config.json new file mode 100644 index 0000000000000..5a5e03e94b24f --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/config.json @@ -0,0 +1,9 @@ +{ + "description": "Tiny/Dummy PyTorch model (text_classification)", + "model_type": "pytorch", + "inference_config": { + "text_classification": { + "classification_labels": ["POSITIVE", "NEGATIVE"] + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/traced_pytorch_model.pt new file mode 100644 index 0000000000000..b2a0ee7c1d47d Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/vocabulary.json new file mode 100644 index 0000000000000..9dc30191a0ca4 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_classification/vocabulary.json @@ -0,0 +1,16 @@ +{ + "vocabulary": [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "Hello", + "world", + "car", + "bike", + "bee", + "bird", + "and" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/config.json new file mode 100644 index 0000000000000..9c5c226cfd232 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/config.json @@ -0,0 +1,7 @@ +{ + "description": "Tiny/Dummy PyTorch model (text_embedding)", + "model_type": "pytorch", + "inference_config": { + "text_embedding": {} + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/traced_pytorch_model.pt new file mode 100644 index 0000000000000..c7d7686060915 Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/vocabulary.json new file mode 100644 index 0000000000000..9dc30191a0ca4 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_text_embedding/vocabulary.json @@ -0,0 +1,16 @@ +{ + "vocabulary": [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "Hello", + "world", + "car", + "bike", + "bee", + "bird", + "and" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/config.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/config.json new file mode 100644 index 0000000000000..ce273c9f19993 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/config.json @@ -0,0 +1,9 @@ +{ + "description": "Tiny/Dummy PyTorch model (zero_shot)", + "model_type": "pytorch", + "inference_config": { + "zero_shot_classification": { + "classification_labels": ["entailment", "neutral", "contradiction"] + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/traced_pytorch_model.pt b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/traced_pytorch_model.pt new file mode 100644 index 0000000000000..acffbb80eaf53 Binary files /dev/null and b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/traced_pytorch_model.pt differ diff --git a/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/vocabulary.json b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/vocabulary.json new file mode 100644 index 0000000000000..a28bce7331749 --- /dev/null +++ b/x-pack/test/functional/services/ml/resources/trained_model_definitions/pt_tiny_zero_shot/vocabulary.json @@ -0,0 +1,24 @@ +{ + "vocabulary": [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "Hello", + "world", + "car", + "bike", + "bee", + "bird", + "and", + "my", + "name", + "is", + "I'm", + "Spartacus", + "glad", + "sad", + "bad" + ] +} \ No newline at end of file diff --git a/x-pack/test/functional/services/ml/trained_models.ts b/x-pack/test/functional/services/ml/trained_models.ts index a15ec9fb1ecd6..c0fb549f97864 100644 --- a/x-pack/test/functional/services/ml/trained_models.ts +++ b/x-pack/test/functional/services/ml/trained_models.ts @@ -7,28 +7,13 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { MlApi } from './api'; import { MlCommonUI } from './common_ui'; -type ModelType = 'regression' | 'classification'; - -export function TrainedModelsProvider( - { getService }: FtrProviderContext, - mlApi: MlApi, - mlCommonUI: MlCommonUI -) { +export function TrainedModelsProvider({ getService }: FtrProviderContext, mlCommonUI: MlCommonUI) { const testSubjects = getService('testSubjects'); const retry = getService('retry'); return { - async createTestTrainedModels( - modelType: ModelType, - count: number = 10, - withIngestPipelines = false - ) { - await mlApi.createTestTrainedModels(modelType, count, withIngestPipelines); - }, - async assertStats(expectedTotalCount: number) { await retry.tryForTime(5 * 1000, async () => { const actualStats = await testSubjects.getVisibleText('mlInferenceModelsStatsBar');