From d73dcab11a37564b0ac07ad0382f6e336722e59a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 19 Feb 2021 10:52:52 +0100 Subject: [PATCH 1/3] enable by default --- src/plugins/discover/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index aebe706f42791..e528e9708bf0d 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -191,7 +191,7 @@ export const uiSettings: Record = { [SEARCH_FIELDS_FROM_SOURCE]: { name: 'Read fields from _source', description: `When enabled will load documents directly from \`_source\`. This is soon going to be deprecated. When disabled, will retrieve fields via the new Fields API in the high-level search service.`, - value: true, + value: false, category: ['discover'], schema: schema.boolean(), }, From d040f50923af7fe9961a1331245c341606ef579b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 14 Jan 2020 15:15:30 +0100 Subject: [PATCH 2/3] [Discover] Fix context view for date_nanos format with custom timestamps (#54089) * Switch from _source to fields when fetching anchor records with date_nanos timestamps * Add testdata * Add functional test --- .../angular/context/api/context.ts | 4 +- .../api/utils/get_es_query_search_after.ts | 23 +++----- test/functional/apps/context/index.js | 1 + .../es_archiver/date_nanos_custom/data.json | 56 +++++++++++++++++++ .../date_nanos_custom/mappings.json | 31 ++++++++++ 5 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 test/functional/fixtures/es_archiver/date_nanos_custom/data.json create mode 100644 test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json diff --git a/src/plugins/discover/public/application/angular/context/api/context.ts b/src/plugins/discover/public/application/angular/context/api/context.ts index d57b54ca1ea1e..43f6e83d286b3 100644 --- a/src/plugins/discover/public/application/angular/context/api/context.ts +++ b/src/plugins/discover/public/application/angular/context/api/context.ts @@ -64,9 +64,9 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFields const searchSource = await createSearchSource(indexPattern, filters); const sortDirToApply = type === 'successors' ? sortDir : reverseSortDir(sortDir); - const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor._source[timeField]) : ''; + const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor.fields[timeField][0]) : ''; const timeValueMillis = - nanos !== '' ? convertIsoToMillis(anchor._source[timeField]) : anchor.sort[0]; + nanos !== '' ? convertIsoToMillis(anchor.fields[timeField][0]) : anchor.sort[0]; const intervals = generateIntervals(LOOKUP_OFFSETS, timeValueMillis, type, sortDir); let documents: EsHitRecordList = []; diff --git a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts index 1f745ab1b728e..6ff220390b7cc 100644 --- a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts +++ b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts @@ -7,6 +7,7 @@ */ import { SurrDocType, EsHitRecordList, EsHitRecord } from '../context'; +import { convertIsoToNanosAsStr } from './date_conversion'; export type EsQuerySearchAfter = [string | number, string | number]; @@ -28,23 +29,15 @@ export function getEsQuerySearchAfter( // already surrounding docs -> first or last record is used const afterTimeRecIdx = type === 'successors' && documents.length ? documents.length - 1 : 0; const afterTimeDoc = documents[afterTimeRecIdx]; - let afterTimeValue = afterTimeDoc.sort[0]; - if (nanoSeconds) { - afterTimeValue = useNewFieldsApi - ? afterTimeDoc.fields[timeFieldName][0] - : afterTimeDoc._source[timeFieldName]; - } + const afterTimeValue = nanoSeconds + ? convertIsoToNanosAsStr(afterTimeDoc.fields[timeFieldName][0]) + : afterTimeDoc.sort[0]; return [afterTimeValue, afterTimeDoc.sort[1]]; } // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser // ES search_after also works when number is provided as string - const searchAfter = new Array(2) as EsQuerySearchAfter; - searchAfter[0] = anchor.sort[0]; - if (nanoSeconds) { - searchAfter[0] = useNewFieldsApi - ? anchor.fields[timeFieldName][0] - : anchor._source[timeFieldName]; - } - searchAfter[1] = anchor.sort[1]; - return searchAfter; + return [ + nanoSeconds ? convertIsoToNanosAsStr(anchor.fields[timeFieldName][0]) : anchor.sort[0], + anchor.sort[1], + ]; } diff --git a/test/functional/apps/context/index.js b/test/functional/apps/context/index.js index bc6b892d8aa0e..245f88a337dce 100644 --- a/test/functional/apps/context/index.js +++ b/test/functional/apps/context/index.js @@ -32,5 +32,6 @@ export default function ({ getService, getPageObjects, loadTestFile }) { loadTestFile(require.resolve('./_filters')); loadTestFile(require.resolve('./_size')); loadTestFile(require.resolve('./_date_nanos')); + loadTestFile(require.resolve('./_date_nanos_custom_timestamp')); }); } diff --git a/test/functional/fixtures/es_archiver/date_nanos_custom/data.json b/test/functional/fixtures/es_archiver/date_nanos_custom/data.json new file mode 100644 index 0000000000000..73cba70a8b93d --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_custom/data.json @@ -0,0 +1,56 @@ +{ + "type": "doc", + "value": { + "id": "index-pattern:date_nanos_custom_timestamp", + "index": ".kibana", + "source": { + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"test\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"test.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"test\"}}},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date_nanos\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "date_nanos_custom_timestamp" + }, + "references": [ + ], + "type": "index-pattern", + "updated_at": "2020-01-09T21:43:20.283Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 00:30:04.828740" + } + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 08:30:04.828733" + } + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 00:30:04.828723" + } + } +} + + diff --git a/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json new file mode 100644 index 0000000000000..98af509c4e6f2 --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json @@ -0,0 +1,31 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "date_nanos_custom_timestamp", + "mappings": { + "properties": { + "test": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "timestamp": { + "format": "yyyy-MM-dd HH:mm:ss.SSSSSS", + "type": "date_nanos" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} From 705379fb7710fc56c66cb6237d8641de4169eafa Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 19 Feb 2021 14:31:16 +0100 Subject: [PATCH 3/3] fix search_after --- .../api/utils/get_es_query_search_after.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts index 6ff220390b7cc..1f745ab1b728e 100644 --- a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts +++ b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts @@ -7,7 +7,6 @@ */ import { SurrDocType, EsHitRecordList, EsHitRecord } from '../context'; -import { convertIsoToNanosAsStr } from './date_conversion'; export type EsQuerySearchAfter = [string | number, string | number]; @@ -29,15 +28,23 @@ export function getEsQuerySearchAfter( // already surrounding docs -> first or last record is used const afterTimeRecIdx = type === 'successors' && documents.length ? documents.length - 1 : 0; const afterTimeDoc = documents[afterTimeRecIdx]; - const afterTimeValue = nanoSeconds - ? convertIsoToNanosAsStr(afterTimeDoc.fields[timeFieldName][0]) - : afterTimeDoc.sort[0]; + let afterTimeValue = afterTimeDoc.sort[0]; + if (nanoSeconds) { + afterTimeValue = useNewFieldsApi + ? afterTimeDoc.fields[timeFieldName][0] + : afterTimeDoc._source[timeFieldName]; + } return [afterTimeValue, afterTimeDoc.sort[1]]; } // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser // ES search_after also works when number is provided as string - return [ - nanoSeconds ? convertIsoToNanosAsStr(anchor.fields[timeFieldName][0]) : anchor.sort[0], - anchor.sort[1], - ]; + const searchAfter = new Array(2) as EsQuerySearchAfter; + searchAfter[0] = anchor.sort[0]; + if (nanoSeconds) { + searchAfter[0] = useNewFieldsApi + ? anchor.fields[timeFieldName][0] + : anchor._source[timeFieldName]; + } + searchAfter[1] = anchor.sort[1]; + return searchAfter; }