Skip to content

Commit

Permalink
[ML] Extends support for anomaly charts when model plot is enabled (#…
Browse files Browse the repository at this point in the history 
…34079) (#34159)

* [ML] Extends support for anomaly charts when model plot is enabled

* [ML] Edits to util functions following review
  • Loading branch information
@peteharverson
peteharverson authored Mar 29, 2019
1 parent 4e9cdd9 commit 0e4d0a3
Show file tree
Hide file tree
Showing 14 changed files with 505 additions and 120 deletions.
47 changes: 47 additions & 0 deletions x-pack/plugins/ml/common/util/__tests__/anomaly_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
getMultiBucketImpactLabel,
getEntityFieldName,
getEntityFieldValue,
getEntityFieldList,
showActualForFunction,
showTypicalForFunction,
isRuleSupported,
Expand Down Expand Up @@ -349,6 +350,52 @@ describe('ML - anomaly utils', () => {

});

describe('getEntityFieldList', () => {
it('returns an empty list for a record with no by, over or partition fields', () => {
expect(getEntityFieldList(noEntityRecord)).to.be.empty();
});

it('returns correct list for a record with a by field', () => {
expect(getEntityFieldList(byEntityRecord)).to.eql([
{
fieldName: 'airline',
fieldValue: 'JZA',
fieldType: 'by'
}
]);
});

it('returns correct list for a record with a partition field', () => {
expect(getEntityFieldList(partitionEntityRecord)).to.eql([
{
fieldName: 'airline',
fieldValue: 'AAL',
fieldType: 'partition'
}
]);
});

it('returns correct list for a record with an over field', () => {
expect(getEntityFieldList(overEntityRecord)).to.eql([
{
fieldName: 'clientip',
fieldValue: '37.157.32.164',
fieldType: 'over'
}
]);
});

it('returns correct list for a record with a by and over field', () => {
expect(getEntityFieldList(rareEntityRecord)).to.eql([
{
fieldName: 'clientip',
fieldValue: '173.252.74.112',
fieldType: 'over'
}
]);
});
});

describe('showActualForFunction', () => {
it('returns true for expected function descriptions', () => {
expect(showActualForFunction('count')).to.be(true);
Expand Down
183 changes: 141 additions & 42 deletions x-pack/plugins/ml/common/util/__tests__/job_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import {
calculateDatafeedFrequencyDefaultSeconds,
isTimeSeriesViewJob,
isTimeSeriesViewDetector,
isTimeSeriesViewFunction,
isSourceDataChartableForDetector,
isModelPlotChartableForDetector,
getPartitioningFieldNames,
isModelPlotEnabled,
isJobVersionGte,
Expand Down Expand Up @@ -158,47 +159,145 @@ describe('ML - job utils', () => {

});

describe('isTimeSeriesViewFunction', () => {

it('returns true for expected functions', () => {
expect(isTimeSeriesViewFunction('count')).to.be(true);
expect(isTimeSeriesViewFunction('low_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_count')).to.be(true);
expect(isTimeSeriesViewFunction('non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('low_non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('low_distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('metric')).to.be(true);
expect(isTimeSeriesViewFunction('mean')).to.be(true);
expect(isTimeSeriesViewFunction('low_mean')).to.be(true);
expect(isTimeSeriesViewFunction('high_mean')).to.be(true);
expect(isTimeSeriesViewFunction('median')).to.be(true);
expect(isTimeSeriesViewFunction('low_median')).to.be(true);
expect(isTimeSeriesViewFunction('high_median')).to.be(true);
expect(isTimeSeriesViewFunction('min')).to.be(true);
expect(isTimeSeriesViewFunction('max')).to.be(true);
expect(isTimeSeriesViewFunction('sum')).to.be(true);
expect(isTimeSeriesViewFunction('low_sum')).to.be(true);
expect(isTimeSeriesViewFunction('high_sum')).to.be(true);
expect(isTimeSeriesViewFunction('non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('low_non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('high_non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('rare')).to.be(true);
});

it('returns false for expected functions', () => {
expect(isTimeSeriesViewFunction('freq_rare')).to.be(false);
expect(isTimeSeriesViewFunction('info_content')).to.be(false);
expect(isTimeSeriesViewFunction('low_info_content')).to.be(false);
expect(isTimeSeriesViewFunction('high_info_content')).to.be(false);
expect(isTimeSeriesViewFunction('varp')).to.be(false);
expect(isTimeSeriesViewFunction('low_varp')).to.be(false);
expect(isTimeSeriesViewFunction('high_varp')).to.be(false);
expect(isTimeSeriesViewFunction('time_of_day')).to.be(false);
expect(isTimeSeriesViewFunction('time_of_week')).to.be(false);
expect(isTimeSeriesViewFunction('lat_long')).to.be(false);
describe('isSourceDataChartableForDetector', () => {

const job = {
analysis_config: {
detectors: [
{ function: 'count' }, // 0
{ function: 'low_count' }, // 1
{ function: 'high_count' }, // 2
{ function: 'non_zero_count' }, // 3
{ function: 'low_non_zero_count' }, // 4
{ function: 'high_non_zero_count' }, // 5
{ function: 'distinct_count' }, // 6
{ function: 'low_distinct_count' }, // 7
{ function: 'high_distinct_count' }, // 8
{ function: 'metric' }, // 9
{ function: 'mean' }, // 10
{ function: 'low_mean' }, // 11
{ function: 'high_mean' }, // 12
{ function: 'median' }, // 13
{ function: 'low_median' }, // 14
{ function: 'high_median' }, // 15
{ function: 'min' }, // 16
{ function: 'max' }, // 17
{ function: 'sum' }, // 18
{ function: 'low_sum' }, // 19
{ function: 'high_sum' }, // 20
{ function: 'non_null_sum' }, // 21
{ function: 'low_non_null_sum' }, // 22
{ function: 'high_non_null_sum' }, // 23
{ function: 'rare' }, // 24
{ function: 'count', 'by_field_name': 'mlcategory', }, // 25
{ function: 'count', 'by_field_name': 'hrd', }, // 26
{ function: 'freq_rare' }, // 27
{ function: 'info_content' }, // 28
{ function: 'low_info_content' }, // 29
{ function: 'high_info_content' }, // 30
{ function: 'varp' }, // 31
{ function: 'low_varp' }, // 32
{ function: 'high_varp' }, // 33
{ function: 'time_of_day' }, // 34
{ function: 'time_of_week' }, // 35
{ function: 'lat_long' }, // 36
{ function: 'mean', 'field_name': 'NetworkDiff' }, //37
]
},
datafeed_config: {
script_fields: {
hrd: {
script: {
inline: 'return domainSplit(doc["query"].value, params).get(1);',
lang: 'painless'
}
},
NetworkDiff: {
script: {
source: 'doc["NetworkOut"].value - doc["NetworkIn"].value',
lang: 'painless'
}
}
}
}
};

it('returns true for expected detectors', () => {
expect(isSourceDataChartableForDetector(job, 0)).to.be(true);
expect(isSourceDataChartableForDetector(job, 1)).to.be(true);
expect(isSourceDataChartableForDetector(job, 2)).to.be(true);
expect(isSourceDataChartableForDetector(job, 3)).to.be(true);
expect(isSourceDataChartableForDetector(job, 4)).to.be(true);
expect(isSourceDataChartableForDetector(job, 5)).to.be(true);
expect(isSourceDataChartableForDetector(job, 6)).to.be(true);
expect(isSourceDataChartableForDetector(job, 7)).to.be(true);
expect(isSourceDataChartableForDetector(job, 8)).to.be(true);
expect(isSourceDataChartableForDetector(job, 9)).to.be(true);
expect(isSourceDataChartableForDetector(job, 10)).to.be(true);
expect(isSourceDataChartableForDetector(job, 11)).to.be(true);
expect(isSourceDataChartableForDetector(job, 12)).to.be(true);
expect(isSourceDataChartableForDetector(job, 13)).to.be(true);
expect(isSourceDataChartableForDetector(job, 14)).to.be(true);
expect(isSourceDataChartableForDetector(job, 15)).to.be(true);
expect(isSourceDataChartableForDetector(job, 16)).to.be(true);
expect(isSourceDataChartableForDetector(job, 17)).to.be(true);
expect(isSourceDataChartableForDetector(job, 18)).to.be(true);
expect(isSourceDataChartableForDetector(job, 19)).to.be(true);
expect(isSourceDataChartableForDetector(job, 20)).to.be(true);
expect(isSourceDataChartableForDetector(job, 21)).to.be(true);
expect(isSourceDataChartableForDetector(job, 22)).to.be(true);
expect(isSourceDataChartableForDetector(job, 23)).to.be(true);
expect(isSourceDataChartableForDetector(job, 24)).to.be(true);
});

it('returns false for expected detectors', () => {
expect(isSourceDataChartableForDetector(job, 25)).to.be(false);
expect(isSourceDataChartableForDetector(job, 26)).to.be(false);
expect(isSourceDataChartableForDetector(job, 27)).to.be(false);
expect(isSourceDataChartableForDetector(job, 28)).to.be(false);
expect(isSourceDataChartableForDetector(job, 29)).to.be(false);
expect(isSourceDataChartableForDetector(job, 30)).to.be(false);
expect(isSourceDataChartableForDetector(job, 31)).to.be(false);
expect(isSourceDataChartableForDetector(job, 32)).to.be(false);
expect(isSourceDataChartableForDetector(job, 33)).to.be(false);
expect(isSourceDataChartableForDetector(job, 34)).to.be(false);
expect(isSourceDataChartableForDetector(job, 35)).to.be(false);
expect(isSourceDataChartableForDetector(job, 36)).to.be(false);
expect(isSourceDataChartableForDetector(job, 37)).to.be(false);
});
});

describe('isModelPlotChartableForDetector', () => {
const job1 = {
analysis_config: {
detectors: [
{ function: 'count' }
]
}
};

const job2 = {
analysis_config: {
detectors: [
{ function: 'count' },
{ function: 'info_content' }
]
},
model_plot_config: {
enabled: true
}
};

it('returns false when model plot is not enabled', () => {
expect(isModelPlotChartableForDetector(job1, 0)).to.be(false);
});

it('returns true for count detector when model plot is enabled', () => {
expect(isModelPlotChartableForDetector(job2, 0)).to.be(true);
});

it('returns true for info_content detector when model plot is enabled', () => {
expect(isModelPlotChartableForDetector(job2, 1)).to.be(true);
});
});

Expand Down
34 changes: 34 additions & 0 deletions x-pack/plugins/ml/common/util/anomaly_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,40 @@ export function getEntityFieldValue(record) {
return undefined;
}

// Returns the list of partitioning entity fields for the source record as a list
// of objects in the form { fieldName: airline, fieldValue: AAL, fieldType: partition }
export function getEntityFieldList(record) {
const entityFields = [];
if (record.partition_field_name !== undefined) {
entityFields.push({
fieldName: record.partition_field_name,
fieldValue: record.partition_field_value,
fieldType: 'partition'
});
}

if (record.over_field_name !== undefined) {
entityFields.push({
fieldName: record.over_field_name,
fieldValue: record.over_field_value,
fieldType: 'over'
});
}

// For jobs with by and over fields, don't add the 'by' field as this
// field will only be added to the top-level fields for record type results
// if it also an influencer over the bucket.
if (record.by_field_name !== undefined && record.over_field_name === undefined) {
entityFields.push({
fieldName: record.by_field_name,
fieldValue: record.by_field_value,
fieldType: 'by'
});
}

return entityFields;
}

// Returns whether actual values should be displayed for a record with the specified function description.
// Note that the 'function' field in a record contains what the user entered e.g. 'high_count',
// whereas the 'function_description' field holds a ML-built display hint for function e.g. 'count'.
Expand Down
Loading

0 comments on commit 0e4d0a3

Please sign in to comment.