[Maps] Add support for date fields in dynamic styles (#47903)

* [Maps] retrieve geo_point value from docvalue_fields instead of _source

* [Maps] retrieve geo_point value from docvalue_fields instead of _source

* Dynamically style by time field

* Add epoch_millis to date fields in docvalues

* add functional test ensuring _search request only pulls what is needed

* Add functional test for dynamic styles by date field

* Support dynamic styles in top hits agg

* Add getTimeFields fn to vector_source

* Retrieve only epoch_millis for date fields in docvalues

* use new Fields syntax, simplify docvalue_fields creation

* fix comment

Author: nickpeihl

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js

@@ -97,6 +97,17 @@ export class ESSearchSource extends AbstractESSource {
     }
   }
 
+  async getDateFields() {
+    try {
+      const indexPattern = await this._getIndexPattern();
+      return indexPattern.fields.getByType('date').map(field => {
+        return { name: field.name, label: field.name };
+      });
+    } catch (error) {
+      return [];
+    }
+  }
+
   getMetricFields() {
     return [];
   }
@@ -158,6 +169,28 @@ export class ESSearchSource extends AbstractESSource {
     ];
   }
 
+  async _excludeDateFields(fieldNames) {
+    const dateFieldNames = _.map(await this.getDateFields(), 'name');
+    return fieldNames.filter(field => {
+      return !dateFieldNames.includes(field);
+    });
+  }
+
+  // Returns docvalue_fields array for the union of indexPattern's dateFields and request's field names.
+  async _getDateDocvalueFields(searchFields) {
+    const dateFieldNames = _.map(await this.getDateFields(), 'name');
+    return searchFields
+      .filter(fieldName => {
+        return dateFieldNames.includes(fieldName);
+      })
+      .map(fieldName => {
+        return {
+          field: fieldName,
+          format: 'epoch_millis'
+        };
+      });
+  }
+
   async _getTopHits(layerName, searchFilters, registerCancelCallback) {
     const {
       topHitsSplitField,
@@ -183,16 +216,19 @@ export class ESSearchSource extends AbstractESSource {
     const topHits = {
       size: topHitsSize,
       script_fields: scriptFields,
+      docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames),
     };
+    const nonDateFieldNames = await this._excludeDateFields(searchFilters.fieldNames);
+
     if (this._hasSort()) {
       topHits.sort = this._buildEsSort();
     }
     if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) {
       topHits._source = false;
-      topHits.docvalue_fields = searchFilters.fieldNames;
+      topHits.docvalue_fields.push(...nonDateFieldNames);
     } else {
       topHits._source = {
-        includes: searchFilters.fieldNames
+        includes: nonDateFieldNames
       };
     }
 
@@ -238,22 +274,23 @@ export class ESSearchSource extends AbstractESSource {
   // searchFilters.fieldNames contains geo field and any fields needed for styling features
   // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size
   async _getSearchHits(layerName, searchFilters, registerCancelCallback) {
+    const initialSearchContext = {
+      docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames)
+    };
     const geoField = await this._getGeoField();
 
     let searchSource;
     if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) {
       // Request geo_point and style fields in docvalue_fields insted of _source
       // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source
       // 2) Setting _source to false so we avoid pulling back unneeded fields.
-      const initialSearchContext = {
-        docvalue_fields: searchFilters.fieldNames
-      };
+      initialSearchContext.docvalue_fields.push(...(await this._excludeDateFields(searchFilters.fieldNames)));
       searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext);
       searchSource.setField('source', false); // do not need anything from _source
       searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
     } else {
       // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source
-      searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT);
+      searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext);
       // Setting "fields" instead of "source: { includes: []}"
       // because SearchSource automatically adds the following by default
       // 1) all scripted fields

x-pack/legacy/plugins/maps/public/layers/sources/es_source.js

@@ -323,7 +323,7 @@ export class AbstractESSource extends AbstractVectorSource {
       return null;
     }
 
-    const fieldFromIndexPattern = indexPattern.fields.byName[rawFieldName];
+    const fieldFromIndexPattern = indexPattern.fields.getByName(rawFieldName);
     if (!fieldFromIndexPattern) {
       return null;
     }

x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js

@@ -80,6 +80,10 @@ export class AbstractVectorSource extends AbstractSource {
     return null;
   }
 
+  async getDateFields() {
+    return [];
+  }
+
   async getNumberFields() {
     return [];
   }

x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js

@@ -403,7 +403,7 @@ export class VectorStyle extends AbstractStyle {
           } else if (range.delta === 0) {//values are identical
             styleValue = 1;//snap to end of color range
           } else {
-            styleValue = (feature.properties[name] - range.min) / range.delta;
+            styleValue = (value - range.min) / range.delta;
           }
         } else {
           if (isNaN(value)) {

x-pack/legacy/plugins/maps/public/layers/vector_layer.js

@@ -254,6 +254,14 @@ export class VectorLayer extends AbstractLayer {
   }
 
   async getOrdinalFields() {
+    const timeFields = await this._source.getDateFields();
+    const timeFieldOptions = timeFields.map(({ label, name }) => {
+      return {
+        label,
+        name,
+        origin: SOURCE_DATA_ID_ORIGIN
+      };
+    });
     const numberFields = await this._source.getNumberFields();
     const numberFieldOptions = numberFields.map(({ label, name }) => {
       return {
@@ -273,7 +281,7 @@ export class VectorLayer extends AbstractLayer {
       joinFields.push(...fields);
     });
 
-    return [...numberFieldOptions, ...joinFields];
+    return [...timeFieldOptions, ...numberFieldOptions, ...joinFields];
   }
 
   getIndexPatternIds() {

x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js

@@ -43,5 +43,15 @@ export default function ({ getPageObjects, getService }) {
       expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates,bytes,hour_of_day');
     });
 
+    it('should format date fields as epoch_millis when data driven styling is applied to a date field', async () => {
+      await PageObjects.maps.loadSavedMap('document example with data driven styles on date field');
+      const response = await getResponse();
+      const firstHit = response.hits.hits[0];
+      expect(Object.keys(firstHit).join(',')).to.equal('_index,_type,_id,_score,fields');
+      expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates,bytes,@timestamp');
+      expect(firstHit.fields['@timestamp']).to.be.an('array');
+      expect(firstHit.fields['@timestamp'][0]).to.equal('1442709321445');
+    });
+
   });
 }

x-pack/test/functional/es_archives/maps/kibana/data.json

@@ -334,6 +334,62 @@
   }
 }
 
+{
+  "type": "doc",
+  "value": {
+    "id": "map:c9277dd0-eb8f-11e9-ae47-693d6a50fb9e",
+    "index": ".kibana",
+    "source": {
+      "map": {
+        "title" : "document example with data driven styles on date field",
+        "description" : "",
+        "mapStateJSON": "{\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-20T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filters\":[]}",
+        "layerListJSON": "[{\"id\":\"0hmz5\",\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"road_map\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{}},\"type\":\"VECTOR_TILE\",\"minZoom\":0,\"maxZoom\":24},{\"id\":\"z52lq\",\"label\":\"logstash\",\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"e1a5e1a6-676c-4a89-8ea9-0d91d64b73c6\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"showTooltip\":true,\"tooltipProperties\":[],\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"field\":{\"label\":\"@timestamp\",\"name\":\"@timestamp\",\"origin\":\"source\"}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":4,\"maxSize\":24,\"field\":{\"label\":\"bytes\",\"name\":\"bytes\",\"origin\":\"source\"}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbol\":{\"options\":{\"symbolizeAs\":\"circle\",\"symbolId\":\"airfield\"}}}},\"type\":\"VECTOR\"}]",
+        "uiStateJSON": "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}",
+        "bounds" : {
+          "type" : "Polygon",
+          "coordinates" : [
+            [
+              [
+                -132.42879,
+                44.5711
+              ],
+              [
+                -132.42879,
+                20.22627
+              ],
+              [
+                -68.79303,
+                20.22627
+              ],
+              [
+                -68.79303,
+                44.5711
+              ],
+              [
+                -132.42879,
+                44.5711
+              ]
+            ]
+          ]
+        }
+      },
+      "type" : "map",
+      "references" : [
+        {
+          "name" : "layer_1_source_index_pattern",
+          "type" : "index-pattern",
+          "id" : "c698b940-e149-11e8-a35a-370a8516603a"
+        }
+      ],
+      "migrationVersion" : {
+        "map" : "7.5.0"
+      },
+      "updated_at" : "2019-10-10T18:57:19.916Z"
+    }
+  }
+}
+
 {
   "type": "doc",
   "value": {