From 1bbbcc5fec89212f247cfd679d48d6839a318e00 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 4 Oct 2019 16:31:25 -0600 Subject: [PATCH 01/11] [Maps] retrieve geo_point value from docvalue_fields instead of _source --- .../maps/public/elasticsearch_geo_utils.js | 62 +++++-------------- .../public/elasticsearch_geo_utils.test.js | 55 ++++------------ .../es_search_source/es_search_source.js | 49 +++++++++++---- .../maps/public/layers/sources/es_source.js | 4 +- 4 files changed, 64 insertions(+), 106 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js index bfb3f710116e0..10ef1ee517c3c 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js @@ -7,7 +7,6 @@ import _ from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse } from 'wellknown'; -import { decodeGeoHash } from 'ui/utils/decode_geo_hash'; import { DECIMAL_DEGREES_PRECISION, ES_GEO_FIELD_TYPE, @@ -91,62 +90,29 @@ export function hitsToGeoJson(hits, flattenHit, geoFieldName, geoFieldType) { }; } -function pointGeometryFactory(lat, lon) { - return { - type: GEO_JSON_TYPE.POINT, - coordinates: [lon, lat] - }; -} - +// Parse geo_point docvalue_field +// Either +// 1) Array of latLon strings +// 2) latLon string export function geoPointToGeometry(value, accumulator) { if (!value) { return; } - if (typeof value === 'string') { - - const commaSplit = value.split(','); - let point; - if (commaSplit.length === 1) { - const geohash = decodeGeoHash(value); - point = pointGeometryFactory(geohash.latitude[2], geohash.longitude[2]); - } else { - const lat = parseFloat(commaSplit[0]); - const lon = parseFloat(commaSplit[1]); - point = pointGeometryFactory(lat, lon); + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + geoPointToGeometry(value[i], accumulator); } - accumulator.push(point); - return; - } - - if (typeof value === 'object' && _.has(value, 'lat') && _.has(value, 'lon')) { - accumulator.push(pointGeometryFactory(value.lat, value.lon)); return; } - if (!Array.isArray(value)) { - const errorMessage = i18n.translate('xpack.maps.es_geo_utils.unsupportedGeoPointValueErrorMessage', { - defaultMessage: `Unsupported geo_point value: {geoPointValue}`, - values: { - geoPointValue: value - } - }); - throw new Error(errorMessage); - } - - if (value.length === 2 - && typeof value[0] === 'number' - && typeof value[1] === 'number') { - const lat = value[LAT_INDEX]; - const lon = value[LON_INDEX]; - accumulator.push(pointGeometryFactory(lat, lon)); - return; - } - - // Geo-point expressed as an array of values - for (let i = 0; i < value.length; i++) { - geoPointToGeometry(value[i], accumulator); - } + const commaSplit = value.split(','); + const lat = parseFloat(commaSplit[0]); + const lon = parseFloat(commaSplit[1]); + accumulator.push({ + type: GEO_JSON_TYPE.POINT, + coordinates: [lon, lat] + }); } export function convertESShapeToGeojsonGeometry(value) { diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js index ab025290127e1..b856feecdbdea 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js @@ -41,12 +41,12 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 } + [geoFieldName]: '20,100' } }, { _source: { - [geoFieldName]: { lat: 30, lon: 110 } + [geoFieldName]: '30,110' } }, ]; @@ -68,7 +68,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 } + [geoFieldName]: '20,100' } }, { @@ -84,7 +84,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 }, + [geoFieldName]: '20,100', myField: 8, }, fields: { @@ -103,8 +103,8 @@ describe('hitsToGeoJson', () => { { _source: { [geoFieldName]: [ - { lat: 20, lon: 100 }, - { lat: 30, lon: 110 } + '20,100', + '30,110' ], myField: 8, } @@ -152,7 +152,7 @@ describe('hitsToGeoJson', () => { { _source: { my: { - location: { lat: 20, lon: 100 }, + location: '20,100', } } } @@ -172,7 +172,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - ['my.location']: { lat: 20, lon: 100 }, + ['my.location']: '20,100', } } ]; @@ -193,7 +193,7 @@ describe('geoPointToGeometry', () => { const lat = 41.12; const lon = -71.34; - it('Should convert value stored as geo-point string', () => { + it('Should convert single docvalue_field', () => { const value = `${lat},${lon}`; const points = []; geoPointToGeometry(value, points); @@ -202,35 +202,11 @@ describe('geoPointToGeometry', () => { expect(points[0].coordinates).toEqual([lon, lat]); }); - it('Should convert value stored as geo-point array', () => { - const value = [lon, lat]; - const points = []; - geoPointToGeometry(value, points); - expect(points.length).toBe(1); - expect(points[0].type).toBe('Point'); - expect(points[0].coordinates).toEqual([lon, lat]); - }); - - it('Should convert value stored as geo-point object', () => { - const value = { - lat, - lon, - }; - const points = []; - geoPointToGeometry(value, points); - expect(points.length).toBe(1); - expect(points[0].type).toBe('Point'); - expect(points[0].coordinates).toEqual([lon, lat]); - }); - - it('Should convert array of values', () => { + it('Should convert multiple docvalue_fields', () => { const lat2 = 30; const lon2 = -60; const value = [ - { - 'lat': lat, - 'lon': lon - }, + `${lat},${lon}`, `${lat2},${lon2}` ]; const points = []; @@ -239,15 +215,6 @@ describe('geoPointToGeometry', () => { expect(points[0].coordinates).toEqual([lon, lat]); expect(points[1].coordinates).toEqual([lon2, lat2]); }); - - it('Should handle point as geohash string', () => { - const geohashValue = 'drm3btev3e86'; - const points = []; - geoPointToGeometry(geohashValue, points); - expect(points.length).toBe(1); - expect(points[0].coordinates).toEqual([-71.34000012651086, 41.12000000663102]); - }); - }); describe('geoShapeToGeometry', () => { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 071e231bb6e2d..948491c4728bf 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -143,6 +143,7 @@ export class ESSearchSource extends AbstractESSource { } = this._descriptor; const indexPattern = await this._getIndexPattern(); + const geoField = await this._getGeoField(); const scriptFields = {}; searchFilters.fieldNames.forEach(fieldName => { @@ -158,7 +159,7 @@ export class ESSearchSource extends AbstractESSource { }); const searchSource = await this._makeSearchSource(searchFilters, 0); - searchSource.setField('aggs', { + const aggs = { entitySplit: { terms: { field: topHitsSplitField, @@ -174,16 +175,22 @@ export class ESSearchSource extends AbstractESSource { } } ], - _source: { - includes: searchFilters.fieldNames - }, size: topHitsSize, script_fields: scriptFields, } } } } - }); + }; + if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { + aggs.entitySplit.aggs.entityHits.top_hits._source = false; + aggs.entitySplit.aggs.entityHits.top_hits.docvalue_fields = searchFilters.fieldNames; + } else { + aggs.entitySplit.aggs.entityHits.top_hits._source = { + includes: searchFilters.fieldNames + }; + } + searchSource.setField('aggs', aggs); const resp = await this._runEsQuery(layerName, searchSource, registerCancelCallback, 'Elasticsearch document top hits request'); @@ -209,14 +216,32 @@ export class ESSearchSource extends AbstractESSource { }; } + // searchFilters.fieldNames contains geo field and any fields needed for styling features + // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size async _getSearchHits(layerName, searchFilters, registerCancelCallback) { - const searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT); - // Setting "fields" instead of "source: { includes: []}" - // because SearchSource automatically adds the following by default - // 1) all scripted fields - // 2) docvalue_fields value is added for each date field in an index - see getComputedFields - // By setting "fields", SearchSource removes all of defaults - searchSource.setField('fields', searchFilters.fieldNames); + const geoField = await this._getGeoField(); + + let searchSource; + if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { + // Request geo_point and style fields in docvalue_fields insted of _source + // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source + // 2) Setting _source to false so we avoid pulling back unneeded fields. + const initialSearchContext = { + docvalue_fields: searchFilters.fieldNames + }; + searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); + searchSource.setField('source', false); // do not need anything from _source + searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields + } else { + // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source + searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT); + // Setting "fields" instead of "source: { includes: []}" + // because SearchSource automatically adds the following by default + // 1) all scripted fields + // 2) docvalue_fields value is added for each date field in an index - see getComputedFields + // By setting "fields", SearchSource removes all of defaults + searchSource.setField('fields', searchFilters.fieldNames); + } const resp = await this._runEsQuery(layerName, searchSource, registerCancelCallback, 'Elasticsearch document request'); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index 48aa9df47b2f4..20cf8baea12c2 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -158,7 +158,7 @@ export class AbstractESSource extends AbstractVectorSource { } } - async _makeSearchSource(searchFilters, limit) { + async _makeSearchSource(searchFilters, limit, initialSearchContext) { const indexPattern = await this._getIndexPattern(); const isTimeAware = await this.isTimeAware(); const applyGlobalQuery = _.get(searchFilters, 'applyGlobalQuery', true); @@ -172,7 +172,7 @@ export class AbstractESSource extends AbstractVectorSource { allFilters.push(timefilter.createFilter(indexPattern, searchFilters.timeFilters)); } - const searchSource = new SearchSource(); + const searchSource = new SearchSource(initialSearchContext); searchSource.setField('index', indexPattern); searchSource.setField('size', limit); searchSource.setField('filter', allFilters); From a6ff011bcd1de017331d403a50d22fc3d7ab340f Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Fri, 4 Oct 2019 16:31:25 -0600 Subject: [PATCH 02/11] [Maps] retrieve geo_point value from docvalue_fields instead of _source --- .../maps/public/elasticsearch_geo_utils.js | 62 +++++-------------- .../public/elasticsearch_geo_utils.test.js | 55 ++++------------ .../es_search_source/es_search_source.js | 51 ++++++++++++--- .../maps/public/layers/sources/es_source.js | 4 +- 4 files changed, 68 insertions(+), 104 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js index bfb3f710116e0..10ef1ee517c3c 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js @@ -7,7 +7,6 @@ import _ from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse } from 'wellknown'; -import { decodeGeoHash } from 'ui/utils/decode_geo_hash'; import { DECIMAL_DEGREES_PRECISION, ES_GEO_FIELD_TYPE, @@ -91,62 +90,29 @@ export function hitsToGeoJson(hits, flattenHit, geoFieldName, geoFieldType) { }; } -function pointGeometryFactory(lat, lon) { - return { - type: GEO_JSON_TYPE.POINT, - coordinates: [lon, lat] - }; -} - +// Parse geo_point docvalue_field +// Either +// 1) Array of latLon strings +// 2) latLon string export function geoPointToGeometry(value, accumulator) { if (!value) { return; } - if (typeof value === 'string') { - - const commaSplit = value.split(','); - let point; - if (commaSplit.length === 1) { - const geohash = decodeGeoHash(value); - point = pointGeometryFactory(geohash.latitude[2], geohash.longitude[2]); - } else { - const lat = parseFloat(commaSplit[0]); - const lon = parseFloat(commaSplit[1]); - point = pointGeometryFactory(lat, lon); + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + geoPointToGeometry(value[i], accumulator); } - accumulator.push(point); - return; - } - - if (typeof value === 'object' && _.has(value, 'lat') && _.has(value, 'lon')) { - accumulator.push(pointGeometryFactory(value.lat, value.lon)); return; } - if (!Array.isArray(value)) { - const errorMessage = i18n.translate('xpack.maps.es_geo_utils.unsupportedGeoPointValueErrorMessage', { - defaultMessage: `Unsupported geo_point value: {geoPointValue}`, - values: { - geoPointValue: value - } - }); - throw new Error(errorMessage); - } - - if (value.length === 2 - && typeof value[0] === 'number' - && typeof value[1] === 'number') { - const lat = value[LAT_INDEX]; - const lon = value[LON_INDEX]; - accumulator.push(pointGeometryFactory(lat, lon)); - return; - } - - // Geo-point expressed as an array of values - for (let i = 0; i < value.length; i++) { - geoPointToGeometry(value[i], accumulator); - } + const commaSplit = value.split(','); + const lat = parseFloat(commaSplit[0]); + const lon = parseFloat(commaSplit[1]); + accumulator.push({ + type: GEO_JSON_TYPE.POINT, + coordinates: [lon, lat] + }); } export function convertESShapeToGeojsonGeometry(value) { diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js index ab025290127e1..b856feecdbdea 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js @@ -41,12 +41,12 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 } + [geoFieldName]: '20,100' } }, { _source: { - [geoFieldName]: { lat: 30, lon: 110 } + [geoFieldName]: '30,110' } }, ]; @@ -68,7 +68,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 } + [geoFieldName]: '20,100' } }, { @@ -84,7 +84,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - [geoFieldName]: { lat: 20, lon: 100 }, + [geoFieldName]: '20,100', myField: 8, }, fields: { @@ -103,8 +103,8 @@ describe('hitsToGeoJson', () => { { _source: { [geoFieldName]: [ - { lat: 20, lon: 100 }, - { lat: 30, lon: 110 } + '20,100', + '30,110' ], myField: 8, } @@ -152,7 +152,7 @@ describe('hitsToGeoJson', () => { { _source: { my: { - location: { lat: 20, lon: 100 }, + location: '20,100', } } } @@ -172,7 +172,7 @@ describe('hitsToGeoJson', () => { const hits = [ { _source: { - ['my.location']: { lat: 20, lon: 100 }, + ['my.location']: '20,100', } } ]; @@ -193,7 +193,7 @@ describe('geoPointToGeometry', () => { const lat = 41.12; const lon = -71.34; - it('Should convert value stored as geo-point string', () => { + it('Should convert single docvalue_field', () => { const value = `${lat},${lon}`; const points = []; geoPointToGeometry(value, points); @@ -202,35 +202,11 @@ describe('geoPointToGeometry', () => { expect(points[0].coordinates).toEqual([lon, lat]); }); - it('Should convert value stored as geo-point array', () => { - const value = [lon, lat]; - const points = []; - geoPointToGeometry(value, points); - expect(points.length).toBe(1); - expect(points[0].type).toBe('Point'); - expect(points[0].coordinates).toEqual([lon, lat]); - }); - - it('Should convert value stored as geo-point object', () => { - const value = { - lat, - lon, - }; - const points = []; - geoPointToGeometry(value, points); - expect(points.length).toBe(1); - expect(points[0].type).toBe('Point'); - expect(points[0].coordinates).toEqual([lon, lat]); - }); - - it('Should convert array of values', () => { + it('Should convert multiple docvalue_fields', () => { const lat2 = 30; const lon2 = -60; const value = [ - { - 'lat': lat, - 'lon': lon - }, + `${lat},${lon}`, `${lat2},${lon2}` ]; const points = []; @@ -239,15 +215,6 @@ describe('geoPointToGeometry', () => { expect(points[0].coordinates).toEqual([lon, lat]); expect(points[1].coordinates).toEqual([lon2, lat2]); }); - - it('Should handle point as geohash string', () => { - const geohashValue = 'drm3btev3e86'; - const points = []; - geoPointToGeometry(geohashValue, points); - expect(points.length).toBe(1); - expect(points[0].coordinates).toEqual([-71.34000012651086, 41.12000000663102]); - }); - }); describe('geoShapeToGeometry', () => { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 53762a1f27d27..2e835951afdf3 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -159,6 +159,7 @@ export class ESSearchSource extends AbstractESSource { } = this._descriptor; const indexPattern = await this._getIndexPattern(); + const geoField = await this._getGeoField(); const scriptFields = {}; searchFilters.fieldNames.forEach(fieldName => { @@ -183,9 +184,13 @@ export class ESSearchSource extends AbstractESSource { aggs: { entityHits: { top_hits: { - _source: { - includes: searchFilters.fieldNames - }, + sort: [ + { + [topHitsTimeField]: { + order: 'desc' + } + } + ], size: topHitsSize, script_fields: scriptFields, } @@ -196,6 +201,14 @@ export class ESSearchSource extends AbstractESSource { if (this._hasSort()) { aggs.entitySplit.aggs.entityHits.top_hits.sort = this._buildEsSort(); } + if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { + aggs.entitySplit.aggs.entityHits.top_hits._source = false; + aggs.entitySplit.aggs.entityHits.top_hits.docvalue_fields = searchFilters.fieldNames; + } else { + aggs.entitySplit.aggs.entityHits.top_hits._source = { + includes: searchFilters.fieldNames + }; + } searchSource.setField('aggs', aggs); const resp = await this._runEsQuery(layerName, searchSource, registerCancelCallback, 'Elasticsearch document top hits request'); @@ -222,14 +235,32 @@ export class ESSearchSource extends AbstractESSource { }; } + // searchFilters.fieldNames contains geo field and any fields needed for styling features + // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size async _getSearchHits(layerName, searchFilters, registerCancelCallback) { - const searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT); - // Setting "fields" instead of "source: { includes: []}" - // because SearchSource automatically adds the following by default - // 1) all scripted fields - // 2) docvalue_fields value is added for each date field in an index - see getComputedFields - // By setting "fields", SearchSource removes all of defaults - searchSource.setField('fields', searchFilters.fieldNames); + const geoField = await this._getGeoField(); + + let searchSource; + if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { + // Request geo_point and style fields in docvalue_fields insted of _source + // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source + // 2) Setting _source to false so we avoid pulling back unneeded fields. + const initialSearchContext = { + docvalue_fields: searchFilters.fieldNames + }; + searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); + searchSource.setField('source', false); // do not need anything from _source + searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields + } else { + // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source + searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT); + // Setting "fields" instead of "source: { includes: []}" + // because SearchSource automatically adds the following by default + // 1) all scripted fields + // 2) docvalue_fields value is added for each date field in an index - see getComputedFields + // By setting "fields", SearchSource removes all of defaults + searchSource.setField('fields', searchFilters.fieldNames); + } if (this._hasSort()) { searchSource.setField('sort', this._buildEsSort()); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index 48aa9df47b2f4..20cf8baea12c2 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -158,7 +158,7 @@ export class AbstractESSource extends AbstractVectorSource { } } - async _makeSearchSource(searchFilters, limit) { + async _makeSearchSource(searchFilters, limit, initialSearchContext) { const indexPattern = await this._getIndexPattern(); const isTimeAware = await this.isTimeAware(); const applyGlobalQuery = _.get(searchFilters, 'applyGlobalQuery', true); @@ -172,7 +172,7 @@ export class AbstractESSource extends AbstractVectorSource { allFilters.push(timefilter.createFilter(indexPattern, searchFilters.timeFilters)); } - const searchSource = new SearchSource(); + const searchSource = new SearchSource(initialSearchContext); searchSource.setField('index', indexPattern); searchSource.setField('size', limit); searchSource.setField('filter', allFilters); From eccced0ed56cba7d3136d44a5c13a7c02bae46cc Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Wed, 25 Sep 2019 15:33:37 -0700 Subject: [PATCH 03/11] Dynamically style by time field --- .../sources/es_search_source/es_search_source.js | 11 +++++++++++ .../legacy/plugins/maps/public/layers/vector_layer.js | 10 +++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 2e835951afdf3..f0dacf0a2becc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -91,6 +91,17 @@ export class ESSearchSource extends AbstractESSource { } } + async getTimeFields() { + try { + const indexPattern = await this._getIndexPattern(); + return indexPattern.fields.byType.date.map(field => { + return { name: field.name, label: field.name }; + }); + } catch (error) { + return []; + } + } + getMetricFields() { return []; } diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js index 829078d98996e..0f6d55544ed83 100644 --- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js +++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js @@ -244,6 +244,14 @@ export class VectorLayer extends AbstractLayer { } async getOrdinalFields() { + const timeFields = await this._source.getTimeFields(); + const timeFieldOptions = timeFields.map(({ label, name }) => { + return { + label, + name, + origin: SOURCE_DATA_ID_ORIGIN + }; + }); const numberFields = await this._source.getNumberFields(); const numberFieldOptions = numberFields.map(({ label, name }) => { return { @@ -263,7 +271,7 @@ export class VectorLayer extends AbstractLayer { joinFields.push(...fields); }); - return [...numberFieldOptions, ...joinFields]; + return [...timeFieldOptions, ...numberFieldOptions, ...joinFields]; } getIndexPatternIds() { From 3cef649c1cbcd504335cfbac65c4e90d485ac89f Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Thu, 10 Oct 2019 08:53:47 -0700 Subject: [PATCH 04/11] Add epoch_millis to date fields in docvalues --- .../es_search_source/es_search_source.js | 17 +++++++++++++---- .../maps/public/layers/styles/vector_style.js | 7 +++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index f0dacf0a2becc..2d6080f067e26 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -166,6 +166,7 @@ export class ESSearchSource extends AbstractESSource { async _getTopHits(layerName, searchFilters, registerCancelCallback) { const { topHitsSplitField, + topHitsTimeField, topHitsSize, } = this._descriptor; @@ -249,22 +250,30 @@ export class ESSearchSource extends AbstractESSource { // searchFilters.fieldNames contains geo field and any fields needed for styling features // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size async _getSearchHits(layerName, searchFilters, registerCancelCallback) { + const initialSearchContext = { + docvalue_fields: [] + }; const geoField = await this._getGeoField(); + const timeFields = await this.getTimeFields(); + timeFields.forEach(field => { + initialSearchContext.docvalue_fields.push({ + field: field.name, + format: 'epoch_millis' + }); + }); let searchSource; if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { // Request geo_point and style fields in docvalue_fields insted of _source // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source // 2) Setting _source to false so we avoid pulling back unneeded fields. - const initialSearchContext = { - docvalue_fields: searchFilters.fieldNames - }; + initialSearchContext.docvalue_fields.push(...searchFilters.fieldNames); searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); searchSource.setField('source', false); // do not need anything from _source searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields } else { // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source - searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT); + searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); // Setting "fields" instead of "source: { includes: []}" // because SearchSource automatically adds the following by default // 1) all scripted fields diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js index f308e371c5aa2..23b54b9fac2bb 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js @@ -394,7 +394,10 @@ export class VectorStyle extends AbstractStyle { for (let j = 0; j < styleFields.length; j++) { const { supportsFeatureState, isScaled, name, range, computedName } = styleFields[j]; - const value = parseFloat(feature.properties[name]); + //Date fields pulled from doc_values is an array of epoch_millis and a date string + const value = Array.isArray(feature.properties[name]) + ? parseFloat(feature.properties[name][0]) + : parseFloat(feature.properties[name]); let styleValue; if (isScaled) { if (isNaN(value) || !range) {//cannot scale @@ -402,7 +405,7 @@ export class VectorStyle extends AbstractStyle { } else if (range.delta === 0) {//values are identical styleValue = 1;//snap to end of color range } else { - styleValue = (feature.properties[name] - range.min) / range.delta; + styleValue = (value - range.min) / range.delta; } } else { if (isNaN(value)) { From 76b51e47c7e354626f39f6e7890f793038987373 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Thu, 10 Oct 2019 11:10:53 -0600 Subject: [PATCH 05/11] add functional test ensuring _search request only pulls what is needed --- .../maps/documents_source/docvalue_fields.js | 47 +++++++++++++++ .../apps/maps/documents_source/index.js | 1 + .../es_archives/maps/kibana/data.json | 58 ++++++++++++++++++- 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js diff --git a/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js b/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js new file mode 100644 index 0000000000000..f083056a6c126 --- /dev/null +++ b/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function ({ getPageObjects, getService }) { + const PageObjects = getPageObjects(['maps']); + const inspector = getService('inspector'); + const testSubjects = getService('testSubjects'); + const log = getService('log'); + + describe('docvalue_fields', () => { + before(async () => { + await PageObjects.maps.loadSavedMap('document example'); + }); + + async function getResponse() { + await inspector.open(); + await inspector.openInspectorRequestsView(); + await testSubjects.click('inspectorRequestDetailResponse'); + const responseBody = await testSubjects.getVisibleText('inspectorResponseBody'); + log.info(responseBody); + await inspector.close(); + return JSON.parse(responseBody); + } + + it('should only fetch geo_point field and nothing else when source does not have data driven styling', async () => { + await PageObjects.maps.loadSavedMap('document example'); + const response = await getResponse(); + const firstHit = response.hits.hits[0]; + expect(Object.keys(firstHit).join(',')).to.equal('_index,_type,_id,_score,fields'); + expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates'); + }); + + it('should only fetch geo_point field and data driven styling fields', async () => { + await PageObjects.maps.loadSavedMap('document example with data driven styles'); + const response = await getResponse(); + const firstHit = response.hits.hits[0]; + expect(Object.keys(firstHit).join(',')).to.equal('_index,_type,_id,_score,fields'); + expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates,bytes,hour_of_day'); + }); + + }); +} diff --git a/x-pack/test/functional/apps/maps/documents_source/index.js b/x-pack/test/functional/apps/maps/documents_source/index.js index ceb97f1a8685c..01a9b37450e24 100644 --- a/x-pack/test/functional/apps/maps/documents_source/index.js +++ b/x-pack/test/functional/apps/maps/documents_source/index.js @@ -6,6 +6,7 @@ export default function ({ loadTestFile }) { describe('documents source', function () { + loadTestFile(require.resolve('./docvalue_fields')); loadTestFile(require.resolve('./search_hits')); loadTestFile(require.resolve('./top_hits')); }); diff --git a/x-pack/test/functional/es_archives/maps/kibana/data.json b/x-pack/test/functional/es_archives/maps/kibana/data.json index 3883d90e5318f..934c1f808a2fe 100644 --- a/x-pack/test/functional/es_archives/maps/kibana/data.json +++ b/x-pack/test/functional/es_archives/maps/kibana/data.json @@ -20,7 +20,7 @@ "index": ".kibana", "source": { "index-pattern": { - "fields": "[{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"@message\"},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"@tags\"},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"agent\"},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"extension\"},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"headings\"},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"host\"},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"index\"},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"links\"},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"machine.os\"},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.article:section\"},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.article:tag\"},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:description\"},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:image\"},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:image:height\"},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:image:width\"},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:site_name\"},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:title\"},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:type\"},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.og:url\"},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.twitter:card\"},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.twitter:description\"},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.twitter:image\"},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.twitter:site\"},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.twitter:title\"},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"relatedContent.url\"},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"request\"},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"response\"},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"spaces\"},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"url\"},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":\"multi\",\"parent\":\"xss\"}]", + "fields" : "[{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"@message\",\"subType\":\"multi\"},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"@tags\",\"subType\":\"multi\"},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent\",\"subType\":\"multi\"},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"extension\",\"subType\":\"multi\"},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"headings\",\"subType\":\"multi\"},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host\",\"subType\":\"multi\"},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"index\",\"subType\":\"multi\"},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"links\",\"subType\":\"multi\"},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"machine.os\",\"subType\":\"multi\"},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.article:section\",\"subType\":\"multi\"},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.article:tag\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:description\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:image\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:image:height\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:image:width\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:site_name\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:title\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:type\",\"subType\":\"multi\"},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.og:url\",\"subType\":\"multi\"},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.twitter:card\",\"subType\":\"multi\"},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.twitter:description\",\"subType\":\"multi\"},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.twitter:image\",\"subType\":\"multi\"},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.twitter:site\",\"subType\":\"multi\"},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.twitter:title\",\"subType\":\"multi\"},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"relatedContent.url\",\"subType\":\"multi\"},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"request\",\"subType\":\"multi\"},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"response\",\"subType\":\"multi\"},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"spaces\",\"subType\":\"multi\"},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"url\",\"subType\":\"multi\"},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"xss\",\"subType\":\"multi\"},{\"name\":\"hour_of_day\",\"type\":\"number\",\"count\":0,\"scripted\":true,\"script\":\"doc['@timestamp'].value.getHour()\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]", "timeFieldName": "@timestamp", "title": "logstash-*" }, @@ -278,6 +278,62 @@ } } +{ + "type": "doc", + "value": { + "id": "map:c9734720-eb7f-11e9-8f42-fb14e91ef4b8", + "index": ".kibana", + "source": { + "map": { + "title" : "document example with data driven styles", + "description" : "", + "mapStateJSON" : "{\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-20T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filters\":[]}", + "layerListJSON" : "[{\"id\":\"0hmz5\",\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"road_map\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{}},\"type\":\"VECTOR_TILE\",\"minZoom\":0,\"maxZoom\":24},{\"id\":\"z52lq\",\"label\":\"logstash\",\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"e1a5e1a6-676c-4a89-8ea9-0d91d64b73c6\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"showTooltip\":true,\"tooltipProperties\":[],\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"field\":{\"label\":\"hour_of_day\",\"name\":\"hour_of_day\",\"origin\":\"source\"}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":4,\"maxSize\":24,\"field\":{\"label\":\"bytes\",\"name\":\"bytes\",\"origin\":\"source\"}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbol\":{\"options\":{\"symbolizeAs\":\"circle\",\"symbolId\":\"airfield\"}}}},\"type\":\"VECTOR\"}]", + "uiStateJSON" : "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}", + "bounds" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -137.80011, + 45.99899 + ], + [ + -137.80011, + 18.31046 + ], + [ + -63.42171, + 18.31046 + ], + [ + -63.42171, + 45.99899 + ], + [ + -137.80011, + 45.99899 + ] + ] + ] + } + }, + "type" : "map", + "references" : [ + { + "name" : "layer_1_source_index_pattern", + "type" : "index-pattern", + "id" : "c698b940-e149-11e8-a35a-370a8516603a" + } + ], + "migrationVersion" : { + "map" : "7.5.0" + }, + "updated_at" : "2019-10-10T17:02:48.465Z" + } + } +} + { "type": "doc", "value": { From 39936a2eabfb83c19aba0b98c7ac465f9a5c62e6 Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Thu, 10 Oct 2019 12:53:45 -0700 Subject: [PATCH 06/11] Add functional test for dynamic styles by date field --- .../maps/documents_source/docvalue_fields.js | 10 ++++ .../es_archives/maps/kibana/data.json | 56 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js b/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js index f083056a6c126..90d4da02e3ed7 100644 --- a/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js +++ b/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js @@ -43,5 +43,15 @@ export default function ({ getPageObjects, getService }) { expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates,bytes,hour_of_day'); }); + it('should format date fields as epoch_millis when data driven styling is applied to a date field', async () => { + await PageObjects.maps.loadSavedMap('document example with data driven styles on date field'); + const response = await getResponse(); + const firstHit = response.hits.hits[0]; + expect(Object.keys(firstHit).join(',')).to.equal('_index,_type,_id,_score,fields'); + expect(Object.keys(firstHit.fields).join(',')).to.equal('geo.coordinates,bytes,@timestamp'); + expect(firstHit.fields['@timestamp']).to.be.an('array'); + expect(firstHit.fields['@timestamp'][0]).to.equal('1442709321445'); + }); + }); } diff --git a/x-pack/test/functional/es_archives/maps/kibana/data.json b/x-pack/test/functional/es_archives/maps/kibana/data.json index 934c1f808a2fe..4b413a9737756 100644 --- a/x-pack/test/functional/es_archives/maps/kibana/data.json +++ b/x-pack/test/functional/es_archives/maps/kibana/data.json @@ -334,6 +334,62 @@ } } +{ + "type": "doc", + "value": { + "id": "map:c9277dd0-eb8f-11e9-ae47-693d6a50fb9e", + "index": ".kibana", + "source": { + "map": { + "title" : "document example with data driven styles on date field", + "description" : "", + "mapStateJSON": "{\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-20T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filters\":[]}", + "layerListJSON": "[{\"id\":\"0hmz5\",\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"road_map\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{}},\"type\":\"VECTOR_TILE\",\"minZoom\":0,\"maxZoom\":24},{\"id\":\"z52lq\",\"label\":\"logstash\",\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"e1a5e1a6-676c-4a89-8ea9-0d91d64b73c6\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"showTooltip\":true,\"tooltipProperties\":[],\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"field\":{\"label\":\"@timestamp\",\"name\":\"@timestamp\",\"origin\":\"source\"}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":4,\"maxSize\":24,\"field\":{\"label\":\"bytes\",\"name\":\"bytes\",\"origin\":\"source\"}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbol\":{\"options\":{\"symbolizeAs\":\"circle\",\"symbolId\":\"airfield\"}}}},\"type\":\"VECTOR\"}]", + "uiStateJSON": "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}", + "bounds" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -132.42879, + 44.5711 + ], + [ + -132.42879, + 20.22627 + ], + [ + -68.79303, + 20.22627 + ], + [ + -68.79303, + 44.5711 + ], + [ + -132.42879, + 44.5711 + ] + ] + ] + } + }, + "type" : "map", + "references" : [ + { + "name" : "layer_1_source_index_pattern", + "type" : "index-pattern", + "id" : "c698b940-e149-11e8-a35a-370a8516603a" + } + ], + "migrationVersion" : { + "map" : "7.5.0" + }, + "updated_at" : "2019-10-10T18:57:19.916Z" + } + } +} + { "type": "doc", "value": { From 3adaec0739e4e442eb66dad3517ddacf6d5ab804 Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Thu, 10 Oct 2019 16:41:12 -0700 Subject: [PATCH 07/11] Support dynamic styles in top hits agg --- .../sources/es_search_source/es_search_source.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index e33d6d60d96ee..cc6dc37cfb191 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -171,6 +171,7 @@ export class ESSearchSource extends AbstractESSource { const indexPattern = await this._getIndexPattern(); const geoField = await this._getGeoField(); + const timeFields = await this.getTimeFields(); const scriptFields = {}; searchFilters.fieldNames.forEach(fieldName => { @@ -188,13 +189,21 @@ export class ESSearchSource extends AbstractESSource { const topHits = { size: topHitsSize, script_fields: scriptFields, + docvalue_fields: [], }; + + timeFields.forEach(field => { + topHits.docvalue_fields.push({ + field: field.name, + format: 'epoch_millis' + }); + }); if (this._hasSort()) { topHits.sort = this._buildEsSort(); } if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { topHits._source = false; - topHits.docvalue_fields = searchFilters.fieldNames; + topHits.docvalue_fields.push(...searchFilters.fieldNames); } else { topHits._source = { includes: searchFilters.fieldNames From c44fa85df76a7e1df459928074ec8eda526f2f9b Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Fri, 11 Oct 2019 12:13:54 -0700 Subject: [PATCH 08/11] Add getTimeFields fn to vector_source --- .../plugins/maps/public/layers/sources/vector_source.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js index a27c5050f1def..6ad2573218d17 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js @@ -80,6 +80,10 @@ export class AbstractVectorSource extends AbstractSource { return null; } + async getTimeFields() { + return []; + } + async getNumberFields() { return []; } From aec93cb7afc7d8ba0901d133faf95e5f11eeb990 Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Mon, 14 Oct 2019 09:00:16 -0700 Subject: [PATCH 09/11] Retrieve only epoch_millis for date fields in docvalues --- .../sources/es_search_source/es_search_source.js | 14 ++++++++++++-- .../maps/public/layers/styles/vector_style.js | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index cc6dc37cfb191..eee4883ab7c11 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -163,6 +163,13 @@ export class ESSearchSource extends AbstractESSource { ]; } + async _getFieldsExcludingTimeFields(fieldNames) { + const timeFieldNames = _.map(await this.getTimeFields(), 'name'); + return fieldNames.filter(field => { + return !timeFieldNames.includes(field); + }); + } + async _getTopHits(layerName, searchFilters, registerCancelCallback) { const { topHitsSplitField, @@ -192,6 +199,8 @@ export class ESSearchSource extends AbstractESSource { docvalue_fields: [], }; + const fieldNames = await this._getFieldsExcludingTimeFields(searchFilters.fieldNames); + timeFields.forEach(field => { topHits.docvalue_fields.push({ field: field.name, @@ -203,7 +212,7 @@ export class ESSearchSource extends AbstractESSource { } if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { topHits._source = false; - topHits.docvalue_fields.push(...searchFilters.fieldNames); + topHits.docvalue_fields.push(...fieldNames); } else { topHits._source = { includes: searchFilters.fieldNames @@ -257,6 +266,7 @@ export class ESSearchSource extends AbstractESSource { }; const geoField = await this._getGeoField(); const timeFields = await this.getTimeFields(); + const fieldNames = await this._getFieldsExcludingTimeFields(searchFilters.fieldNames); timeFields.forEach(field => { initialSearchContext.docvalue_fields.push({ field: field.name, @@ -269,7 +279,7 @@ export class ESSearchSource extends AbstractESSource { // Request geo_point and style fields in docvalue_fields insted of _source // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source // 2) Setting _source to false so we avoid pulling back unneeded fields. - initialSearchContext.docvalue_fields.push(...searchFilters.fieldNames); + initialSearchContext.docvalue_fields.push(...fieldNames); searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); searchSource.setField('source', false); // do not need anything from _source searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js index 23b54b9fac2bb..32251f6c1a877 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js @@ -394,7 +394,7 @@ export class VectorStyle extends AbstractStyle { for (let j = 0; j < styleFields.length; j++) { const { supportsFeatureState, isScaled, name, range, computedName } = styleFields[j]; - //Date fields pulled from doc_values is an array of epoch_millis and a date string + //Date fields pulled from doc_values is an array const value = Array.isArray(feature.properties[name]) ? parseFloat(feature.properties[name][0]) : parseFloat(feature.properties[name]); From 478ae8cb86f6c3e2d8f82bb23e85f25d407e5f7a Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 15 Oct 2019 13:31:40 -0600 Subject: [PATCH 10/11] use new Fields syntax, simplify docvalue_fields creation --- .../es_search_source/es_search_source.js | 53 +++++++++---------- .../maps/public/layers/sources/es_source.js | 2 +- .../public/layers/sources/vector_source.js | 2 +- .../maps/public/layers/styles/vector_style.js | 5 +- .../maps/public/layers/vector_layer.js | 2 +- 5 files changed, 30 insertions(+), 34 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index c894c5364d12c..611e2f149895d 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -97,10 +97,10 @@ export class ESSearchSource extends AbstractESSource { } } - async getTimeFields() { + async getDateFields() { try { const indexPattern = await this._getIndexPattern(); - return indexPattern.fields.byType.date.map(field => { + return indexPattern.fields.getByType('date').map(field => { return { name: field.name, label: field.name }; }); } catch (error) { @@ -169,13 +169,28 @@ export class ESSearchSource extends AbstractESSource { ]; } - async _getFieldsExcludingTimeFields(fieldNames) { - const timeFieldNames = _.map(await this.getTimeFields(), 'name'); + async _excludeDateFields(fieldNames) { + const dateFieldNames = _.map(await this.getDateFields(), 'name'); return fieldNames.filter(field => { - return !timeFieldNames.includes(field); + return !dateFieldNames.includes(field); }); } + // Returns docvalue_field for the union of indexPattern's dateFields and request's field names. + async _getDateDocvalueFields(searchFields) { + const dateFieldNames = _.map(await this.getDateFields(), 'name'); + return searchFields + .filter(fieldName => { + return dateFieldNames.includes(fieldName); + }) + .map(fieldName => { + return { + field: fieldName, + format: 'epoch_millis' + }; + }); + } + async _getTopHits(layerName, searchFilters, registerCancelCallback) { const { topHitsSplitField, @@ -184,7 +199,6 @@ export class ESSearchSource extends AbstractESSource { const indexPattern = await this._getIndexPattern(); const geoField = await this._getGeoField(); - const timeFields = await this.getTimeFields(); const scriptFields = {}; searchFilters.fieldNames.forEach(fieldName => { @@ -202,26 +216,19 @@ export class ESSearchSource extends AbstractESSource { const topHits = { size: topHitsSize, script_fields: scriptFields, - docvalue_fields: [], + docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames), }; + const nonDateFieldNames = await this._excludeDateFields(searchFilters.fieldNames); - const fieldNames = await this._getFieldsExcludingTimeFields(searchFilters.fieldNames); - - timeFields.forEach(field => { - topHits.docvalue_fields.push({ - field: field.name, - format: 'epoch_millis' - }); - }); if (this._hasSort()) { topHits.sort = this._buildEsSort(); } if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { topHits._source = false; - topHits.docvalue_fields.push(...fieldNames); + topHits.docvalue_fields.push(...nonDateFieldNames); } else { topHits._source = { - includes: searchFilters.fieldNames + includes: nonDateFieldNames }; } @@ -268,24 +275,16 @@ export class ESSearchSource extends AbstractESSource { // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size async _getSearchHits(layerName, searchFilters, registerCancelCallback) { const initialSearchContext = { - docvalue_fields: [] + docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames) }; const geoField = await this._getGeoField(); - const timeFields = await this.getTimeFields(); - const fieldNames = await this._getFieldsExcludingTimeFields(searchFilters.fieldNames); - timeFields.forEach(field => { - initialSearchContext.docvalue_fields.push({ - field: field.name, - format: 'epoch_millis' - }); - }); let searchSource; if (geoField.type === ES_GEO_FIELD_TYPE.GEO_POINT) { // Request geo_point and style fields in docvalue_fields insted of _source // 1) Returns geo_point in a consistent format regardless of how geo_point is stored in source // 2) Setting _source to false so we avoid pulling back unneeded fields. - initialSearchContext.docvalue_fields.push(...fieldNames); + initialSearchContext.docvalue_fields.push(...(await this._excludeDateFields(searchFilters.fieldNames))); searchSource = await this._makeSearchSource(searchFilters, ES_SIZE_LIMIT, initialSearchContext); searchSource.setField('source', false); // do not need anything from _source searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index 2e20617848c91..730f640ed3c79 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -323,7 +323,7 @@ export class AbstractESSource extends AbstractVectorSource { return null; } - const fieldFromIndexPattern = indexPattern.fields.byName[rawFieldName]; + const fieldFromIndexPattern = indexPattern.fields.getByName(rawFieldName); if (!fieldFromIndexPattern) { return null; } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js index 6ad2573218d17..b3e8c1b019f28 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/vector_source.js @@ -80,7 +80,7 @@ export class AbstractVectorSource extends AbstractSource { return null; } - async getTimeFields() { + async getDateFields() { return []; } diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js index 00f04b8ef1c0c..f26f4df0b1753 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector_style.js @@ -395,10 +395,7 @@ export class VectorStyle extends AbstractStyle { for (let j = 0; j < styleFields.length; j++) { const { supportsFeatureState, isScaled, name, range, computedName } = styleFields[j]; - //Date fields pulled from doc_values is an array - const value = Array.isArray(feature.properties[name]) - ? parseFloat(feature.properties[name][0]) - : parseFloat(feature.properties[name]); + const value = parseFloat(feature.properties[name]); let styleValue; if (isScaled) { if (isNaN(value) || !range) {//cannot scale diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js index 6c5a1710a3fad..f2c8a05694cdd 100644 --- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js +++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js @@ -254,7 +254,7 @@ export class VectorLayer extends AbstractLayer { } async getOrdinalFields() { - const timeFields = await this._source.getTimeFields(); + const timeFields = await this._source.getDateFields(); const timeFieldOptions = timeFields.map(({ label, name }) => { return { label, From 2ee2145019fa9a1ad2590cd39c1b88bb19633f51 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 15 Oct 2019 13:39:53 -0600 Subject: [PATCH 11/11] fix comment --- .../public/layers/sources/es_search_source/es_search_source.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 611e2f149895d..e5fbde681b308 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -176,7 +176,7 @@ export class ESSearchSource extends AbstractESSource { }); } - // Returns docvalue_field for the union of indexPattern's dateFields and request's field names. + // Returns docvalue_fields array for the union of indexPattern's dateFields and request's field names. async _getDateDocvalueFields(searchFields) { const dateFieldNames = _.map(await this.getDateFields(), 'name'); return searchFields