Skip to content

Commit

Permalink
validate advanced params with explain
Browse files Browse the repository at this point in the history
  • Loading branch information
@alvarezmelissa87
alvarezmelissa87 committed Jun 24, 2020
1 parent d63d6a6 commit 8cc3c34
Show file tree
Hide file tree
Showing 9 changed files with 223 additions and 119 deletions.
7 changes: 7 additions & 0 deletions x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,18 @@ export enum ANALYSIS_CONFIG_TYPE {
}

export enum ANALYSIS_ADVANCED_FIELDS {
ETA = 'eta',
FEATURE_BAG_FRACTION = 'feature_bag_fraction',
FEATURE_INFLUENCE_THRESHOLD = 'feature_influence_threshold',
GAMMA = 'gamma',
LAMBDA = 'lambda',
MAX_TREES = 'max_trees',
METHOD = 'method',
N_NEIGHBORS = 'n_neighbors',
NUM_TOP_CLASSES = 'num_top_classes',
NUM_TOP_FEATURE_IMPORTANCE_VALUES = 'num_top_feature_importance_values',
OUTLIER_FRACTION = 'outlier_fraction',
RANDOMIZE_SEED = 'randomize_seed',
}

export enum OUTLIER_ANALYSIS_METHOD {
Expand Down
73 changes: 69 additions & 4 deletions ..._frame_analytics/pages/analytics_creation/components/advanced_step/advanced_step_form.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/

import React, { FC, Fragment, useMemo } from 'react';
import React, { FC, Fragment, useMemo, useEffect, useState } from 'react';
import {
EuiAccordion,
EuiFieldNumber,
Expand All @@ -23,9 +23,11 @@ import { getModelMemoryLimitErrors } from '../../../analytics_management/hooks/u
import {
ANALYSIS_CONFIG_TYPE,
NUM_TOP_FEATURE_IMPORTANCE_VALUES_MIN,
ANALYSIS_ADVANCED_FIELDS,
} from '../../../../common/analytics';
import { DEFAULT_MODEL_MEMORY_LIMIT } from '../../../analytics_management/hooks/use_create_analytics_form/state';
import { ANALYTICS_STEPS } from '../../page';
import { getExplainData } from '../shared';
import { ContinueButton } from '../continue_button';
import { OutlierHyperParameters } from './outlier_hyper_parameters';

Expand All @@ -38,18 +40,33 @@ export const AdvancedStepForm: FC<CreateAnalyticsStepProps> = ({
state,
setCurrentStep,
}) => {
const [advancedParamErrors, setAdvancedParamErrors] = useState<
{
[key in ANALYSIS_ADVANCED_FIELDS]?: string;
}
>({});

const { setFormState } = actions;
const { form, isJobCreated } = state;
const {
computeFeatureInfluence,
eta,
featureBagFraction,
featureInfluenceThreshold,
gamma,
jobType,
lambda,
maxTrees,
method,
modelMemoryLimit,
modelMemoryLimitValidationResult,
nNeighbors,
numTopClasses,
numTopFeatureImportanceValues,
numTopFeatureImportanceValuesValid,
outlierFraction,
predictionFieldName,
randomizeSeed,
} = form;

const mmlErrors = useMemo(() => getModelMemoryLimitErrors(modelMemoryLimitValidationResult), [
Expand All @@ -61,6 +78,40 @@ export const AdvancedStepForm: FC<CreateAnalyticsStepProps> = ({

const mmlInvalid = modelMemoryLimitValidationResult !== null;

const isStepInvalid = mmlInvalid || Object.keys(advancedParamErrors).length > 0;

useEffect(() => {
(async function () {
const { success, errorMessage } = await getExplainData(form);
const paramErrors: {
[key in ANALYSIS_ADVANCED_FIELDS]?: string;
} = {};

if (!success) {
// Check which field is invalid
Object.values(ANALYSIS_ADVANCED_FIELDS).forEach((param) => {
if (errorMessage.includes(`[${param}]`)) {
paramErrors[param] = errorMessage;
}
});
}
setAdvancedParamErrors(paramErrors);
})();
}, [
eta,
featureBagFraction,
featureInfluenceThreshold,
gamma,
lambda,
maxTrees,
method,
nNeighbors,
numTopClasses,
numTopFeatureImportanceValues,
outlierFraction,
randomizeSeed,
]);

const outlierDetectionAdvancedConfig = (
<Fragment>
<EuiFlexItem style={{ minWidth: '30%' }}>
Expand Down Expand Up @@ -126,6 +177,10 @@ export const AdvancedStepForm: FC<CreateAnalyticsStepProps> = ({
'The minimum outlier score that a document needs to have in order to calculate its feature influence score. Value range: 0-1. Defaults to 0.1.',
}
)}
isInvalid={
advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.FEATURE_INFLUENCE_THRESHOLD] !== undefined
}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.FEATURE_INFLUENCE_THRESHOLD]}
>
<EuiFieldNumber
onChange={(e) =>
Expand Down Expand Up @@ -315,14 +370,24 @@ export const AdvancedStepForm: FC<CreateAnalyticsStepProps> = ({
>
<EuiFlexGroup wrap>
{jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION && (
<OutlierHyperParameters actions={actions} state={state} />
<OutlierHyperParameters
actions={actions}
state={state}
advancedParamErrors={advancedParamErrors}
/>
)}
{isRegOrClassJob && (
<HyperParameters
actions={actions}
state={state}
advancedParamErrors={advancedParamErrors}
/>
)}
{isRegOrClassJob && <HyperParameters actions={actions} state={state} />}
</EuiFlexGroup>
</EuiAccordion>
<EuiSpacer />
<ContinueButton
isDisabled={mmlInvalid}
isDisabled={isStepInvalid}
onClick={() => {
setCurrentStep(ANALYTICS_STEPS.DETAILS);
}}
Expand Down
23 changes: 21 additions & 2 deletions ...ta_frame_analytics/pages/analytics_creation/components/advanced_step/hyper_parameters.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,15 @@ import { EuiFieldNumber, EuiFlexItem, EuiFormRow } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { CreateAnalyticsFormProps } from '../../../analytics_management/hooks/use_create_analytics_form';
import { getNumberValue } from './advanced_step_form';
import { ANALYSIS_ADVANCED_FIELDS } from '../../../../common/analytics';

const MAX_TREES_LIMIT = 2000;

export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }) => {
interface Props extends CreateAnalyticsFormProps {
advancedParamErrors: any;
}

export const HyperParameters: FC<Props> = ({ actions, state, advancedParamErrors }) => {
const { setFormState } = actions;

const { eta, featureBagFraction, gamma, lambda, maxTrees, randomizeSeed } = state.form;
Expand All @@ -28,6 +33,8 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
defaultMessage:
'Regularization parameter to prevent overfitting on the training data set. Must be a non negative value.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.LAMBDA] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.LAMBDA]}
>
<EuiFieldNumber
aria-label={i18n.translate('xpack.ml.dataframe.analytics.create.lambdaInputAriaLabel', {
Expand All @@ -52,6 +59,8 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
helpText={i18n.translate('xpack.ml.dataframe.analytics.create.maxTreesText', {
defaultMessage: 'The maximum number of trees the forest is allowed to contain.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.MAX_TREES] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.MAX_TREES]}
>
<EuiFieldNumber
aria-label={i18n.translate(
Expand Down Expand Up @@ -81,6 +90,8 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
defaultMessage:
'Multiplies a linear penalty associated with the size of individual trees in the forest. Must be non-negative value.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.GAMMA] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.GAMMA]}
>
<EuiFieldNumber
aria-label={i18n.translate('xpack.ml.dataframe.analytics.create.gammaInputAriaLabel', {
Expand All @@ -105,6 +116,8 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
helpText={i18n.translate('xpack.ml.dataframe.analytics.create.etaText', {
defaultMessage: 'The shrinkage applied to the weights. Must be between 0.001 and 1.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.ETA] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.ETA]}
>
<EuiFieldNumber
aria-label={i18n.translate('xpack.ml.dataframe.analytics.create.etaInputAriaLabel', {
Expand All @@ -130,6 +143,10 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
defaultMessage:
'The fraction of features used when selecting a random bag for each candidate split.',
})}
isInvalid={
advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.FEATURE_BAG_FRACTION] !== undefined
}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.FEATURE_BAG_FRACTION]}
>
<EuiFieldNumber
aria-label={i18n.translate(
Expand Down Expand Up @@ -158,12 +175,14 @@ export const HyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }
<EuiFlexItem style={{ minWidth: '30%' }}>
<EuiFormRow
label={i18n.translate('xpack.ml.dataframe.analytics.create.randomizeSeedLabel', {
defaultMessage: 'Randomized seed',
defaultMessage: 'Randomize seed',
})}
helpText={i18n.translate('xpack.ml.dataframe.analytics.create.randomizeSeedText', {
defaultMessage:
'The seed to the random generator that is used to pick which documents will be used for training.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.RANDOMIZE_SEED] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.RANDOMIZE_SEED]}
>
<EuiFieldNumber
aria-label={i18n.translate(
Expand Down
14 changes: 12 additions & 2 deletions ..._analytics/pages/analytics_creation/components/advanced_step/outlier_hyper_parameters.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,15 @@
import React, { FC, Fragment } from 'react';
import { EuiFieldNumber, EuiFlexItem, EuiFormRow, EuiSelect } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { OUTLIER_ANALYSIS_METHOD } from '../../../../common/analytics';
import { OUTLIER_ANALYSIS_METHOD, ANALYSIS_ADVANCED_FIELDS } from '../../../../common/analytics';
import { CreateAnalyticsFormProps } from '../../../analytics_management/hooks/use_create_analytics_form';
import { getNumberValue } from './advanced_step_form';

export const OutlierHyperParameters: FC<CreateAnalyticsFormProps> = ({ actions, state }) => {
interface Props extends CreateAnalyticsFormProps {
advancedParamErrors: any;
}

export const OutlierHyperParameters: FC<Props> = ({ actions, state, advancedParamErrors }) => {
const { setFormState } = actions;

const { method, nNeighbors, outlierFraction, standardizationEnabled } = state.form;
Expand All @@ -27,6 +31,8 @@ export const OutlierHyperParameters: FC<CreateAnalyticsFormProps> = ({ actions,
defaultMessage:
'Sets the method that outlier detection uses. If not set, uses an ensemble of different methods and normalises and combines their individual outlier scores to obtain the overall outlier score. We recommend to use the ensemble method',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.METHOD] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.METHOD]}
>
<EuiSelect
options={Object.values(OUTLIER_ANALYSIS_METHOD).map((outlierMethod) => ({
Expand All @@ -51,6 +57,8 @@ export const OutlierHyperParameters: FC<CreateAnalyticsFormProps> = ({ actions,
defaultMessage:
'The value for how many nearest neighbors each method of outlier detection will use to calculate its outlier score. When not set, different values will be used for different ensemble members. Must be a positive integer',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.N_NEIGHBORS] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.N_NEIGHBORS]}
>
<EuiFieldNumber
aria-label={i18n.translate(
Expand Down Expand Up @@ -79,6 +87,8 @@ export const OutlierHyperParameters: FC<CreateAnalyticsFormProps> = ({ actions,
defaultMessage:
'Sets the proportion of the data set that is assumed to be outlying prior to outlier detection.',
})}
isInvalid={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.OUTLIER_FRACTION] !== undefined}
error={advancedParamErrors[ANALYSIS_ADVANCED_FIELDS.OUTLIER_FRACTION]}
>
<EuiFieldNumber
aria-label={i18n.translate(
Expand Down
Loading

0 comments on commit 8cc3c34

Please sign in to comment.