other and missing bucket support for terms agg (#15525)

src/ui/public/agg_response/hierarchical/_transform_aggregation.js

@@ -9,7 +9,8 @@ export function HierarchicalTransformAggregationProvider() {
         agg,
         parent && parent.aggConfigResult,
         metric.getValue(bucket),
-        agg.getKey(bucket)
+        agg.getKey(bucket),
+        bucket.filters
       );
 
       const branch = {

src/ui/public/agg_response/hierarchical/build_hierarchical_data.js

@@ -77,7 +77,7 @@ export function BuildHierarchicalDataProvider(Private, Notifier) {
       if (!_.isEmpty(displayName)) split.label += ': ' + displayName;
 
       split.tooltipFormatter = tooltipFormatter(raw.columns);
-      const aggConfigResult = new AggConfigResult(firstAgg, null, null, firstAgg.getKey(bucket));
+      const aggConfigResult = new AggConfigResult(firstAgg, null, null, firstAgg.getKey(bucket), bucket.filters);
       split.split = { aggConfig: firstAgg, aggConfigResult: aggConfigResult, key: bucket.key };
       _.each(split.slices.children, function (child) {
         child.aggConfigResult.$parent = aggConfigResult;

src/ui/public/agg_response/tabify/_response_writer.js

@@ -194,7 +194,7 @@ export function TabbedAggResponseWriterProvider(Private) {
         newList.unshift(injected);
       }
 
-      const newAcr = new AggConfigResult(acr.aggConfig, newList[0], acr.value, acr.aggConfig.getKey(acr));
+      const newAcr = new AggConfigResult(acr.aggConfig, newList[0], acr.value, acr.aggConfig.getKey(acr), acr.filters);
       newList.unshift(newAcr);
 
       // and replace the acr in the row buffer if its there
@@ -215,9 +215,9 @@ export function TabbedAggResponseWriterProvider(Private) {
    * @param  {function} block - the function to run while this value is in the row
    * @return {any} - the value that was added
    */
-  TabbedAggResponseWriter.prototype.cell = function (agg, value, block) {
+  TabbedAggResponseWriter.prototype.cell = function (agg, value, block, filters) {
     if (this.asAggConfigResults) {
-      value = new AggConfigResult(agg, this.acrStack[0], value, value);
+      value = new AggConfigResult(agg, this.acrStack[0], value, value, filters);
     }
 
     const staskResult = this.asAggConfigResults && value.type === 'bucket';

src/ui/public/agg_response/tabify/tabify.js

@@ -46,7 +46,7 @@ export function AggResponseTabifyProvider(Private, Notifier) {
             buckets.forEach(function (subBucket, key) {
               write.cell(agg, agg.getKey(subBucket, key), function () {
                 collectBucket(write, subBucket, agg.getKey(subBucket, key), aggScale);
-              });
+              }, subBucket.filters);
             });
           }
         } else if (write.partialRows && write.metricsForAllBuckets && write.minimalColumns) {

src/ui/public/agg_types/__tests__/buckets/_terms_other_bucket_helper.js

@@ -0,0 +1,260 @@
+import expect from 'expect.js';
+import ngMock from 'ng_mock';
+import { OtherBucketHelperProvider } from 'ui/agg_types/buckets/_terms_other_bucket_helper';
+import { VisProvider } from 'ui/vis';
+import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern';
+
+const visConfigSingleTerm = {
+  type: 'pie',
+  aggs: [
+    {
+      type: 'terms',
+      schema: 'segment',
+      params: { field: 'machine.os.raw', otherBucket: true, missingBucket: true }
+    }
+  ]
+};
+
+const visConfigNestedTerm = {
+  type: 'pie',
+  aggs: [
+    {
+      type: 'terms',
+      schema: 'segment',
+      params: { field: 'geo.src', size: 2, otherBucket: false, missingBucket: false }
+    }, {
+      type: 'terms',
+      schema: 'segment',
+      params: { field: 'machine.os.raw', size: 2, otherBucket: true, missingBucket: true }
+    }
+  ]
+};
+
+const singleTermResponse = {
+  'took': 10,
+  'timed_out': false,
+  '_shards': {
+    'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0
+  }, 'hits': {
+    'total': 14005, 'max_score': 0, 'hits': []
+  }, 'aggregations': {
+    '1': {
+      'doc_count_error_upper_bound': 0,
+      'sum_other_doc_count': 8325,
+      'buckets': [
+        { 'key': 'ios', 'doc_count': 2850 },
+        { 'key': 'win xp', 'doc_count': 2830 },
+        { 'key': '__missing__', 'doc_count': 1430 }
+      ]
+    }
+  }, 'status': 200
+};
+
+const nestedTermResponse = {
+  'took': 10,
+  'timed_out': false,
+  '_shards': {
+    'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0
+  }, 'hits': {
+    'total': 14005, 'max_score': 0, 'hits': []
+  }, 'aggregations': {
+    '1': {
+      'doc_count_error_upper_bound': 0,
+      'sum_other_doc_count': 8325,
+      'buckets': [
+        {
+          '2': {
+            'doc_count_error_upper_bound': 0,
+            'sum_other_doc_count': 8325,
+            'buckets': [
+              { 'key': 'ios', 'doc_count': 2850 },
+              { 'key': 'win xp', 'doc_count': 2830 },
+              { 'key': '__missing__', 'doc_count': 1430 }
+            ]
+          },
+          key: 'US',
+          doc_count: 2850
+        }, {
+          '2': {
+            'doc_count_error_upper_bound': 0,
+            'sum_other_doc_count': 8325,
+            'buckets': [
+              { 'key': 'ios', 'doc_count': 1850 },
+              { 'key': 'win xp', 'doc_count': 1830 },
+              { 'key': '__missing__', 'doc_count': 130 }
+            ]
+          },
+          key: 'IN',
+          doc_count: 2830
+        }
+      ]
+    }
+  }, 'status': 200
+};
+
+const singleOtherResponse = {
+  'took': 3,
+  'timed_out': false,
+  '_shards': { 'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0 },
+  'hits': { 'total': 14005, 'max_score': 0, 'hits': [] },
+  'aggregations': {
+    'other-filter': {
+      'buckets': { '': { 'doc_count': 2805 } }
+    }
+  }, 'status': 200
+};
+
+const nestedOtherResponse = {
+  'took': 3,
+  'timed_out': false,
+  '_shards': { 'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0 },
+  'hits': { 'total': 14005, 'max_score': 0, 'hits': [] },
+  'aggregations': {
+    'other-filter': {
+      'buckets': { '-US': { 'doc_count': 2805 }, '-IN': { 'doc_count': 2804 } }
+    }
+  }, 'status': 200
+};
+
+describe('Terms Agg Other bucket helper', () => {
+
+  let otherBucketHelper;
+  let vis;
+
+  function init(aggConfig) {
+    ngMock.module('kibana');
+    ngMock.inject((Private) => {
+      const Vis = Private(VisProvider);
+      const indexPattern = Private(FixturesStubbedLogstashIndexPatternProvider);
+      otherBucketHelper = Private(OtherBucketHelperProvider);
+
+      vis = new Vis(indexPattern, aggConfig);
+    });
+  }
+
+  describe('buildOtherBucketAgg', () => {
+
+    it('returns a function', () => {
+      init(visConfigSingleTerm);
+      const agg = otherBucketHelper.buildOtherBucketAgg(vis.aggs, vis.aggs[0], singleTermResponse);
+      expect(agg).to.be.a('function');
+    });
+
+    it('correctly builds query with single terms agg', () => {
+      init(visConfigSingleTerm);
+      const agg = otherBucketHelper.buildOtherBucketAgg(vis.aggs, vis.aggs[0], singleTermResponse)();
+      const expectedResponse = {
+        aggs: undefined,
+        filters: {
+          filters: {
+            '': {
+              'bool': {
+                'must': [{
+                  'exists': {
+                    'field': 'machine.os.raw',
+                  }
+                }],
+                'filter': [],
+                'should': [],
+                'must_not': [
+                  { 'match_phrase': { 'machine.os.raw': { 'query': 'ios' } } },
+                  { 'match_phrase': { 'machine.os.raw': { 'query': 'win xp' } } }
+                ]
+              }
+            }
+          }
+        }
+      };
+
+      expect(agg['other-filter']).to.eql(expectedResponse);
+    });
+
+    it('correctly builds query for nested terms agg', () => {
+      init(visConfigNestedTerm);
+      const agg = otherBucketHelper.buildOtherBucketAgg(vis.aggs, vis.aggs[1], nestedTermResponse)();
+      const expectedResponse = {
+        'other-filter': {
+          aggs: undefined,
+          'filters': {
+            'filters': {
+              '-IN': {
+                'bool': {
+                  'must': [
+                    { match_phrase: { 'geo.src': { 'query': 'IN' } } },
+                    {
+                      'exists': {
+                        'field': 'machine.os.raw',
+                      }
+                    }
+                  ], 'filter': [],
+                  'should': [],
+                  'must_not': [
+                    { 'match_phrase': { 'machine.os.raw': { 'query': 'ios' } } },
+                    { 'match_phrase': { 'machine.os.raw': { 'query': 'win xp' } } }
+                  ]
+                }
+              }, '-US': {
+                'bool': {
+                  'must': [
+                    { 'match_phrase': { 'geo.src': { 'query': 'US' } } },
+                    {
+                      'exists': {
+                        'field': 'machine.os.raw',
+                      }
+                    }
+                  ], 'filter': [], 'should': [], 'must_not': [
+                    { 'match_phrase': { 'machine.os.raw': { 'query': 'ios' } } },
+                    { 'match_phrase': { 'machine.os.raw': { 'query': 'win xp' } } }
+                  ]
+                }
+              }
+            }
+          }
+        }
+      };
+
+      expect(agg).to.eql(expectedResponse);
+    });
+  });
+
+  describe('mergeOtherBucketAggResponse', () => {
+    it('correctly merges other bucket with single terms agg', () => {
+      init(visConfigSingleTerm);
+      const otherAggConfig = otherBucketHelper.buildOtherBucketAgg(vis.aggs, vis.aggs[0], singleTermResponse)();
+      const mergedResponse = otherBucketHelper
+        .mergeOtherBucketAggResponse(vis.aggs, singleTermResponse, singleOtherResponse, vis.aggs[0], otherAggConfig);
+
+      expect(mergedResponse.aggregations['1'].buckets[3].key).to.equal('Other');
+      expect(mergedResponse.aggregations['1'].buckets[3].filters.length).to.equal(2);
+    });
+
+    it('correctly merges other bucket with nested terms agg', () => {
+      init(visConfigNestedTerm);
+      const otherAggConfig = otherBucketHelper.buildOtherBucketAgg(vis.aggs, vis.aggs[1], nestedTermResponse)();
+      const mergedResponse = otherBucketHelper
+        .mergeOtherBucketAggResponse(vis.aggs, nestedTermResponse, nestedOtherResponse, vis.aggs[1], otherAggConfig);
+
+      expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).to.equal('Other');
+      expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].filters.length).to.equal(2);
+    });
+
+  });
+
+  describe('updateMissingBucket', () => {
+    it('correctly updates missing bucket key', () => {
+      init(visConfigNestedTerm);
+      const updatedResponse = otherBucketHelper.updateMissingBucket(singleTermResponse, vis.aggs, vis.aggs[0]);
+      expect(updatedResponse.aggregations['1'].buckets.find(bucket => bucket.key === 'Missing')).to.not.be('undefined');
+    });
+
+    it('correctly sets the bucket filter', () => {
+      const updatedResponse = otherBucketHelper.updateMissingBucket(singleTermResponse, vis.aggs, vis.aggs[0]);
+      const missingBucket = updatedResponse.aggregations['1'].buckets.find(bucket => bucket.key === 'Missing');
+      expect(missingBucket.filters).to.not.be('undefined');
+      expect(missingBucket.filters[0]).to.eql({
+        meta: { index: 'logstash-*', negate: true },
+        exists: { field: 'geo.src' }
+      });
+    });
+  });
+});

src/ui/public/agg_types/__tests__/index.js

@@ -5,6 +5,7 @@ import './agg_params';
 import './buckets/_histogram';
 import './buckets/_geo_hash';
 import './buckets/_range';
+import './buckets/_terms_other_bucket_helper';
 import './buckets/date_histogram/_editor';
 import './buckets/date_histogram/_params';
 import { AggTypesIndexProvider } from 'ui/agg_types/index';

src/ui/public/agg_types/agg_type.js

@@ -140,6 +140,18 @@ export function AggTypesAggTypeProvider(Private) {
      */
     this.decorateAggConfig = config.decorateAggConfig || null;
 
+    /**
+     * A function that needs to be called after the main request has been made
+     * and should return an updated response
+     * @param aggConfigs - agg config array used to produce main request
+     * @param aggConfig - AggConfig that requested the post flight request
+     * @param searchSourceAggs - SearchSource aggregation configuration
+     * @param resp - Response to the main request
+     * @param nestedSearchSource - the new SearchSource that will be used to make post flight request
+     * @return {Promise}
+     */
+    this.postFlightRequest = config.postFlightRequest || _.identity;
+
     if (config.getFormat) {
       this.getFormat = config.getFormat;
     }