[ML] Data Frame Analytics saved search creation functional tests (ela…

…stic#125040)

* create filter saved search test for each dfa job type

* add search query test cases

* temp comment of failing with insuff memory test case

* remove problem case and update test text

* Use downsampled farequote archive

* remove unnecessary file

Co-authored-by: Robert Oskamp <robert.oskamp@elastic.co>

x-pack/test/functional/apps/ml/data_frame_analytics/index.ts

@@ -16,5 +16,8 @@ export default function ({ loadTestFile }: FtrProviderContext) {
  loadTestFile(require.resolve('./classification_creation'));
  loadTestFile(require.resolve('./cloning'));
  loadTestFile(require.resolve('./feature_importance'));
+ loadTestFile(require.resolve('./regression_creation_saved_search'));
+ loadTestFile(require.resolve('./classification_creation_saved_search'));
+ loadTestFile(require.resolve('./outlier_detection_creation_saved_search'));
  });
 }

x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation_saved_search.ts

@@ -0,0 +1,333 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AnalyticsTableRowDetails } from '../../../services/ml/data_frame_analytics_table';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ getService }: FtrProviderContext) {
+ const esArchiver = getService('esArchiver');
+ const ml = getService('ml');
+ const editedDescription = 'Edited description';
+
+ describe('regression saved search creation', function () {
+ before(async () => {
+ await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote_small');
+ await ml.testResources.createIndexPatternIfNeeded('ft_farequote_small', '@timestamp');
+ await ml.testResources.createSavedSearchFarequoteLuceneIfNeeded('ft_farequote_small');
+ await ml.testResources.createSavedSearchFarequoteKueryIfNeeded('ft_farequote_small');
+ await ml.testResources.setKibanaTimeZoneToUTC();
+
+ await ml.securityUI.loginAsMlPowerUser();
+ });
+
+ after(async () => {
+ await ml.api.cleanMlIndices();
+ await ml.testResources.deleteSavedSearches();
+ await ml.testResources.deleteIndexPatternByTitle('ft_farequote_small');
+ });
+
+ const dateNow = Date.now();
+ const testDataList = [
+ {
+ suiteTitle: 'with lucene query',
+ jobType: 'regression',
+ jobId: `fq_saved_search_2_${dateNow}`,
+ jobDescription: 'Regression job based on a saved search with lucene query',
+ source: 'ft_farequote_lucene',
+ get destinationIndex(): string {
+ return `user-${this.jobId}`;
+ },
+ runtimeFields: {
+ uppercase_airline: {
+ type: 'keyword',
+ script: 'emit(params._source.airline.toUpperCase())',
+ },
+ },
+ dependentVariable: 'responsetime',
+ trainingPercent: 20,
+ modelMemory: '20mb',
+ createIndexPattern: true,
+ expected: {
+ source: 'ft_farequote_small',
+ runtimeFieldsEditorContent: ['{', ' "uppercase_airline": {', ' "type": "keyword",'],
+ row: {
+ memoryStatus: 'ok',
+ type: 'regression',
+ status: 'stopped',
+ progress: '100',
+ },
+ rowDetails: {
+ jobDetails: [
+ {
+ section: 'state',
+ expectedEntries: {
+ id: `fq_saved_search_2_${dateNow}`,
+ state: 'stopped',
+ data_counts:
+ '{"training_docs_count":320,"test_docs_count":1284,"skipped_docs_count":0}',
+ description: 'Regression job based on a saved search with lucene query',
+ },
+ },
+ { section: 'progress', expectedEntries: { Phase: '8/8' } },
+ ],
+ } as AnalyticsTableRowDetails,
+ },
+ },
+ {
+ suiteTitle: 'with kuery query',
+ jobType: 'regression',
+ jobId: `fq_saved_search_3_${dateNow}`,
+ jobDescription: 'Regression job based on a saved search with kuery query',
+ source: 'ft_farequote_kuery',
+ get destinationIndex(): string {
+ return `user-${this.jobId}`;
+ },
+ runtimeFields: {
+ uppercase_airline: {
+ type: 'keyword',
+ script: 'emit(params._source.airline.toUpperCase())',
+ },
+ },
+ dependentVariable: 'responsetime',
+ trainingPercent: 20,
+ modelMemory: '20mb',
+ createIndexPattern: true,
+ expected: {
+ source: 'ft_farequote_small',
+ runtimeFieldsEditorContent: ['{', ' "uppercase_airline": {', ' "type": "keyword",'],
+ row: {
+ memoryStatus: 'ok',
+ type: 'regression',
+ status: 'stopped',
+ progress: '100',
+ },
+ rowDetails: {
+ jobDetails: [
+ {
+ section: 'state',
+ expectedEntries: {
+ id: `fq_saved_search_3_${dateNow}`,
+ state: 'stopped',
+ data_counts:
+ '{"training_docs_count":320,"test_docs_count":1283,"skipped_docs_count":0}',
+ description: 'Regression job based on a saved search with kuery query',
+ },
+ },
+ { section: 'progress', expectedEntries: { Phase: '8/8' } },
+ ],
+ } as AnalyticsTableRowDetails,
+ },
+ },
+ ];
+
+ for (const testData of testDataList) {
+ describe(`${testData.suiteTitle}`, function () {
+ after(async () => {
+ await ml.api.deleteIndices(testData.destinationIndex);
+ await ml.testResources.deleteIndexPatternByTitle(testData.destinationIndex);
+ });
+
+ it('loads the data frame analytics wizard', async () => {
+ await ml.testExecution.logTestStep('loads the data frame analytics page');
+ await ml.navigation.navigateToMl();
+ await ml.navigation.navigateToDataFrameAnalytics();
+
+ await ml.testExecution.logTestStep('loads the source selection modal');
+
+ // Disable anti-aliasing to stabilize canvas image rendering assertions
+ await ml.commonUI.disableAntiAliasing();
+
+ await ml.dataFrameAnalytics.startAnalyticsCreation();
+
+ await ml.testExecution.logTestStep(
+ 'selects the source data and loads the job wizard page'
+ );
+ await ml.jobSourceSelection.selectSourceForAnalyticsJob(testData.source);
+ await ml.dataFrameAnalyticsCreation.assertConfigurationStepActive();
+ });
+
+ it('navigates through the wizard and sets all needed fields', async () => {
+ await ml.testExecution.logTestStep('selects the job type');
+ await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists();
+ await ml.dataFrameAnalyticsCreation.selectJobType(testData.jobType);
+
+ await ml.testExecution.logTestStep('displays the runtime mappings editor switch');
+ await ml.dataFrameAnalyticsCreation.assertRuntimeMappingSwitchExists();
+
+ await ml.testExecution.logTestStep('enables the runtime mappings editor');
+ await ml.dataFrameAnalyticsCreation.toggleRuntimeMappingsEditorSwitch(true);
+ await ml.dataFrameAnalyticsCreation.assertRuntimeMappingsEditorContent(['']);
+
+ await ml.testExecution.logTestStep('sets runtime mappings');
+ await ml.dataFrameAnalyticsCreation.setRuntimeMappingsEditorContent(
+ JSON.stringify(testData.runtimeFields)
+ );
+ await ml.dataFrameAnalyticsCreation.applyRuntimeMappings();
+ await ml.dataFrameAnalyticsCreation.assertRuntimeMappingsEditorContent(
+ testData.expected.runtimeFieldsEditorContent
+ );
+
+ await ml.testExecution.logTestStep('inputs the dependent variable');
+ await ml.dataFrameAnalyticsCreation.assertDependentVariableInputExists();
+ await ml.dataFrameAnalyticsCreation.selectDependentVariable(testData.dependentVariable);
+
+ await ml.testExecution.logTestStep('inputs the training percent');
+ await ml.dataFrameAnalyticsCreation.assertTrainingPercentInputExists();
+ await ml.dataFrameAnalyticsCreation.setTrainingPercent(testData.trainingPercent);
+
+ await ml.testExecution.logTestStep('displays the source data preview');
+ await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists();
+
+ await ml.testExecution.logTestStep('displays the include fields selection');
+ await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
+
+ await ml.testExecution.logTestStep('continues to the additional options step');
+ await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
+
+ await ml.testExecution.logTestStep('accepts the suggested model memory limit');
+ await ml.dataFrameAnalyticsCreation.assertModelMemoryInputExists();
+ await ml.dataFrameAnalyticsCreation.assertModelMemoryInputPopulated();
+
+ await ml.testExecution.logTestStep('continues to the details step');
+ await ml.dataFrameAnalyticsCreation.continueToDetailsStep();
+
+ await ml.testExecution.logTestStep('inputs the job id');
+ await ml.dataFrameAnalyticsCreation.assertJobIdInputExists();
+ await ml.dataFrameAnalyticsCreation.setJobId(testData.jobId);
+
+ await ml.testExecution.logTestStep('inputs the job description');
+ await ml.dataFrameAnalyticsCreation.assertJobDescriptionInputExists();
+ await ml.dataFrameAnalyticsCreation.setJobDescription(testData.jobDescription);
+
+ await ml.testExecution.logTestStep(
+ 'should default the set destination index to job id switch to true'
+ );
+ await ml.dataFrameAnalyticsCreation.assertDestIndexSameAsIdSwitchExists();
+ await ml.dataFrameAnalyticsCreation.assertDestIndexSameAsIdCheckState(true);
+
+ await ml.testExecution.logTestStep('should input the destination index');
+ await ml.dataFrameAnalyticsCreation.setDestIndexSameAsIdCheckState(false);
+ await ml.dataFrameAnalyticsCreation.assertDestIndexInputExists();
+ await ml.dataFrameAnalyticsCreation.setDestIndex(testData.destinationIndex);
+
+ await ml.testExecution.logTestStep('continues to the validation step');
+ await ml.dataFrameAnalyticsCreation.continueToValidationStep();
+
+ await ml.testExecution.logTestStep('checks validation callouts exist');
+ await ml.dataFrameAnalyticsCreation.assertValidationCalloutsExists();
+ await ml.dataFrameAnalyticsCreation.assertAllValidationCalloutsPresent(3);
+
+ await ml.testExecution.logTestStep('continues to the create step');
+ await ml.dataFrameAnalyticsCreation.continueToCreateStep();
+
+ await ml.testExecution.logTestStep('sets the create data view switch');
+ await ml.dataFrameAnalyticsCreation.assertCreateIndexPatternSwitchExists();
+ await ml.dataFrameAnalyticsCreation.setCreateIndexPatternSwitchState(
+ testData.createIndexPattern
+ );
+ });
+
+ it('runs the analytics job and displays it correctly in the job list', async () => {
+ await ml.testExecution.logTestStep('creates and starts the analytics job');
+ await ml.dataFrameAnalyticsCreation.assertCreateButtonExists();
+ await ml.dataFrameAnalyticsCreation.assertStartJobCheckboxCheckState(true);
+ await ml.dataFrameAnalyticsCreation.createAnalyticsJob(testData.jobId);
+
+ await ml.testExecution.logTestStep('finishes analytics processing');
+ await ml.dataFrameAnalytics.waitForAnalyticsCompletion(testData.jobId);
+
+ await ml.testExecution.logTestStep('displays the analytics table');
+ await ml.dataFrameAnalyticsCreation.navigateToJobManagementPage();
+ await ml.dataFrameAnalytics.assertAnalyticsTableExists();
+
+ await ml.testExecution.logTestStep('displays the stats bar');
+ await ml.dataFrameAnalytics.assertAnalyticsStatsBarExists();
+
+ await ml.testExecution.logTestStep('displays the created job in the analytics table');
+ await ml.dataFrameAnalyticsTable.refreshAnalyticsTable();
+ await ml.dataFrameAnalyticsTable.filterWithSearchString(testData.jobId, 1);
+
+ await ml.testExecution.logTestStep(
+ 'displays details for the created job in the analytics table'
+ );
+ await ml.dataFrameAnalyticsTable.assertAnalyticsRowFields(testData.jobId, {
+ id: testData.jobId,
+ description: testData.jobDescription,
+ memoryStatus: testData.expected.row.memoryStatus,
+ sourceIndex: testData.expected.source,
+ destinationIndex: testData.destinationIndex,
+ type: testData.expected.row.type,
+ status: testData.expected.row.status,
+ progress: testData.expected.row.progress,
+ });
+ await ml.dataFrameAnalyticsTable.assertAnalyticsRowDetails(
+ testData.jobId,
+ testData.expected.rowDetails
+ );
+ });
+
+ it('edits the analytics job and displays it correctly in the job list', async () => {
+ await ml.testExecution.logTestStep(
+ 'should open the edit form for the created job in the analytics table'
+ );
+ await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+ await ml.testExecution.logTestStep('should input the description in the edit form');
+ await ml.dataFrameAnalyticsEdit.assertJobDescriptionEditInputExists();
+ await ml.dataFrameAnalyticsEdit.setJobDescriptionEdit(editedDescription);
+
+ await ml.testExecution.logTestStep(
+ 'should input the model memory limit in the edit form'
+ );
+ await ml.dataFrameAnalyticsEdit.assertJobMmlEditInputExists();
+ await ml.dataFrameAnalyticsEdit.setJobMmlEdit('21mb');
+
+ await ml.testExecution.logTestStep('should submit the edit job form');
+ await ml.dataFrameAnalyticsEdit.updateAnalyticsJob();
+
+ await ml.testExecution.logTestStep(
+ 'displays details for the edited job in the analytics table'
+ );
+ await ml.dataFrameAnalyticsTable.assertAnalyticsRowFields(testData.jobId, {
+ id: testData.jobId,
+ description: editedDescription,
+ memoryStatus: testData.expected.row.memoryStatus,
+ sourceIndex: testData.expected.source,
+ destinationIndex: testData.destinationIndex,
+ type: testData.expected.row.type,
+ status: testData.expected.row.status,
+ progress: testData.expected.row.progress,
+ });
+
+ await ml.testExecution.logTestStep(
+ 'creates the destination index and writes results to it'
+ );
+ await ml.api.assertIndicesExist(testData.destinationIndex);
+ await ml.api.assertIndicesNotEmpty(testData.destinationIndex);
+
+ await ml.testExecution.logTestStep('displays the results view for created job');
+ await ml.dataFrameAnalyticsTable.openResultsView(testData.jobId);
+ await ml.dataFrameAnalyticsResults.assertRegressionEvaluatePanelElementsExists();
+ await ml.dataFrameAnalyticsResults.assertRegressionTablePanelExists();
+ await ml.dataFrameAnalyticsResults.assertResultsTableExists();
+ await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist();
+ await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
+
+ await ml.commonUI.resetAntiAliasing();
+ });
+
+ it('displays the analytics job in the map view', async () => {
+ await ml.testExecution.logTestStep('should open the map view for created job');
+ await ml.navigation.navigateToDataFrameAnalytics();
+ await ml.dataFrameAnalyticsTable.openMapView(testData.jobId);
+ await ml.dataFrameAnalyticsMap.assertMapElementsExists();
+ await ml.dataFrameAnalyticsMap.assertJobMapTitle(testData.jobId);
+ });
+ });
+ }
+ });
+}

x-pack/test/functional/apps/ml/index.ts

@@ -25,6 +25,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
  await ml.securityCommon.cleanMlRoles();
 
  await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote');
+ await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote_small');
  await esArchiver.unload('x-pack/test/functional/es_archives/ml/ecommerce');
  await esArchiver.unload('x-pack/test/functional/es_archives/ml/categorization_small');
  await esArchiver.unload('x-pack/test/functional/es_archives/ml/event_rate_nanos');

x-pack/test/functional/es_archives/ml/farequote_small/mappings.json

@@ -0,0 +1,48 @@
+{
+ "type": "index",
+ "value": {
+ "aliases": {
+ },
+ "index": "ft_farequote_small",
+ "mappings": {
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "@version": {
+ "fields": {
+ "keyword": {
+ "ignore_above": 256,
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ },
+ "airline": {
+ "type": "keyword"
+ },
+ "responsetime": {
+ "type": "float"
+ },
+ "sourcetype": {
+ "type": "keyword"
+ },
+ "type": {
+ "fields": {
+ "keyword": {
+ "ignore_above": 256,
+ "type": "keyword"
+ }
+ },
+ "type": "text"
+ }
+ }
+ },
+ "settings": {
+ "index": {
+ "number_of_replicas": "1",
+ "number_of_shards": "1"
+ }
+ }
+ }
+}