[ML] Data transformation fixes for rare/population chart.

x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_rare.js

@@ -76,6 +76,9 @@ export class ExplorerChartRare extends React.Component {
 
     // Left margin is adjusted later for longest y-axis label.
     const margin = { top: 10, right: 0, bottom: 30, left: 0 };
+    if (config.functionDescription === 'count') {
+      margin.left = 60;
+    }
 
     let lineChartXScale = null;
     let lineChartYScale = null;
@@ -85,10 +88,13 @@ export class ExplorerChartRare extends React.Component {
     const CHART_TYPE = (config.functionDescription === 'rare') ? 'rare' : 'population';
     const CHART_Y_ATTRIBUTE = (config.functionDescription === 'rare') ? 'entity' : 'value';
 
+    let highlight = config.chartData.find(d => (d.anomalyScore !== undefined));
+    highlight = highlight && highlight.entity;
+
     const filteredChartData = init(config);
     drawRareChart(filteredChartData);
 
-    function init({ chartLimits, chartData }) {
+    function init({ chartData }) {
       const $el = $('.ml-explorer-chart');
 
       // Clear any existing elements from the visualization,
@@ -104,8 +110,6 @@ export class ExplorerChartRare extends React.Component {
         .attr('width', svgWidth)
         .attr('height', svgHeight);
 
-      let highlight = chartData.find(d => (d.anomalyScore !== undefined));
-      highlight = highlight && highlight.entity;
       const categoryLimit = 30;
       const scaleCategories = d3.nest()
         .key(d => d.entity)
@@ -114,21 +118,32 @@ export class ExplorerChartRare extends React.Component {
           return b.values.length - a.values.length;
         })
         .filter((d, i) => {
+          // only filter for rare charts
+          if (CHART_TYPE === 'rare') {
+            return true;
+          }
           return (i < categoryLimit || d.key === highlight);
         })
         .map(d => d.key);
 
       chartData = chartData.filter((d) => {
-        return (scaleCategories.findIndex(s => (s === d.entity)) > -1);
+        return (scaleCategories.includes(d.entity));
       });
 
       if (CHART_TYPE === 'population') {
+        const focusData = chartData.filter((d) => {
+          return d.entity === highlight;
+        }).map(d => d.value);
+        const focusExtent = d3.extent(focusData);
+
+        // now again filter chartData to include only the data points within the domain
+        chartData = chartData.filter((d) => {
+          return (d.value <= focusExtent[1]);
+        });
+
         lineChartYScale = d3.scale.linear()
           .range([chartHeight, 0])
-          .domain([
-            chartLimits.min,
-            chartLimits.max
-          ])
+          .domain([0, focusExtent[1]])
           .nice();
       } else if (CHART_TYPE === 'rare') {
         // avoid overflowing the border of the highlighted area
@@ -239,7 +254,6 @@ export class ExplorerChartRare extends React.Component {
         xAxis.ticks(numTicksForDateFormat(vizWidth, xAxisTickFormat));
       }
 
-
       const yAxis = d3.svg.axis().scale(lineChartYScale)
         .orient('left')
         .innerTickSize(0)
@@ -277,7 +291,11 @@ export class ExplorerChartRare extends React.Component {
       const dots = dotGroup.selectAll('circle').data(dotsData);
 
       dots.enter().append('circle')
-        .attr('r', radius);
+        .classed('values-dots-circle', true)
+        .classed('values-dots-circle-blur', (d) => {
+          return (d.entity !== highlight);
+        })
+        .attr('r', d => ((d.entity === highlight) ? (radius * 1.5) : radius));
 
       dots
         .attr('cx', rareChartValuesLine.x())

x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js

@@ -138,15 +138,28 @@ export function explorerChartsContainerServiceFactory(
       );
     }
 
-    // Query 4 - load the rare chart's event distribution
+    // Query 4 - load context data distribution
     function getEventDistribution(config, range) {
-      let splitField = config.entityFields.find(f => f.fieldType === 'partition');
+      // Defaults to use the "partition" field to group the data
+      let splitField;
       let filterField = null;
+
+      // If this is a detector using the rare function,
+      // use the "by" field to split the data and filter by the selection partition.
       if (config.functionDescription === 'rare') {
         splitField = config.entityFields.find(f => f.fieldType === 'by');
         filterField = config.entityFields.find(f => f.fieldType === 'partition');
       }
 
+      // Logic to consider an over field: If the above didn't result in choosing a split Field,
+      // check if an over field is part of the detector, if yes, use that to split and
+      // filter by a possible partition field.
+      const overField = config.entityFields.find(f => f.fieldType === 'over');
+      if (splitField === undefined && overField !== undefined) {
+        splitField = overField;
+        filterField = config.entityFields.find(f => f.fieldType === 'partition');
+      }
+
       const datafeedQuery = _.get(config, 'datafeedConfig.query', null);
       return mlResultsService.getEventDistributionData(
         config.datafeedConfig.indices,
@@ -190,7 +203,7 @@ export function explorerChartsContainerServiceFactory(
 
       let chartData;
       if (eventDistribution.length > 0) {
-        const filterField = records[0].by_field_value;
+        const filterField = records[0].by_field_value || records[0].over_field_value;
         chartData = eventDistribution.filter(d => (d.entity !== filterField));
         _.map(metricData, (value, time) => {
           if (value > 0) {
@@ -267,9 +280,9 @@ export function explorerChartsContainerServiceFactory(
     function findNearestChartPointToTime(chartData, record) {
       const time = record[ML_TIME_FIELD_NAME];
 
-      if (record.function === 'rare' || record.function === 'count') {
+      if (record.function === 'rare' || record.over_field_value !== 'undefined') {
         chartData = chartData.filter((d) => {
-          return d.entity === record.by_field_value;
+          return d.entity === (record.by_field_value || record.over_field_value);
         });
       }
 

x-pack/plugins/ml/public/explorer/explorer_charts/styles/explorer_chart.less

@@ -51,11 +51,16 @@
     stroke-width: 2;
   }
 
-  .values-dots circle {
+  .values-dots circle,
+  .values-dots-circle {
     fill: #32a7c2;
     stroke-width: 0;
   }
 
+  .values-dots circle.values-dots-circle-blur {
+    fill: #aaa;
+  }
+
   .metric-value {
     opacity: 1;
     fill: #32a7c2;

x-pack/plugins/ml/public/services/results_service.js

@@ -1414,7 +1414,7 @@ function getEventDistributionData(
   interval) {
   return new Promise((resolve, reject) => {
     // only get this data for count (used by rare chart)
-    if (metricFunction !== 'count') {
+    if (metricFunction !== 'count' || splitField === undefined) {
       return resolve([]);
     }