[Discover] Fix context view for date_nanos format with custom timesta…

…mps (elastic#54089)

* Switch from _source to fields when fetching anchor records with date_nanos timestamps

* Add testdata

* Add functional test

src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context/api/context.ts

@@ -74,9 +74,9 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) {
     const searchSource = await createSearchSource(indexPattern, filters);
     const sortDirToApply = type === 'successors' ? sortDir : reverseSortDir(sortDir);
 
-    const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor._source[timeField]) : '';
+    const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor.fields[timeField][0]) : '';
     const timeValueMillis =
-      nanos !== '' ? convertIsoToMillis(anchor._source[timeField]) : anchor.sort[0];
+      nanos !== '' ? convertIsoToMillis(anchor.fields[timeField][0]) : anchor.sort[0];
 
     const intervals = generateIntervals(LOOKUP_OFFSETS, timeValueMillis, type, sortDir);
     let documents: EsHitRecordList = [];

src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context/api/utils/get_es_query_search_after.ts

@@ -39,14 +39,14 @@ export function getEsQuerySearchAfter(
     const afterTimeRecIdx = type === 'successors' && documents.length ? documents.length - 1 : 0;
     const afterTimeDoc = documents[afterTimeRecIdx];
     const afterTimeValue = nanoSeconds
-      ? convertIsoToNanosAsStr(afterTimeDoc._source[timeFieldName])
+      ? convertIsoToNanosAsStr(afterTimeDoc.fields[timeFieldName][0])
       : afterTimeDoc.sort[0];
     return [afterTimeValue, afterTimeDoc.sort[1]];
   }
   // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser
   // ES search_after also works when number is provided as string
   return [
-    nanoSeconds ? convertIsoToNanosAsStr(anchor._source[timeFieldName]) : anchor.sort[0],
+    nanoSeconds ? convertIsoToNanosAsStr(anchor.fields[timeFieldName][0]) : anchor.sort[0],
     anchor.sort[1],
   ];
 }

test/functional/apps/context/_date_nanos_custom_timestamp.js

@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+
+const TEST_INDEX_PATTERN = 'date_nanos_custom_timestamp';
+const TEST_DEFAULT_CONTEXT_SIZE = 1;
+const TEST_STEP_SIZE = 3;
+
+export default function({ getService, getPageObjects }) {
+  const kibanaServer = getService('kibanaServer');
+  const docTable = getService('docTable');
+  const PageObjects = getPageObjects(['common', 'context', 'timePicker', 'discover']);
+  const esArchiver = getService('esArchiver');
+
+  describe('context view for date_nanos with custom timestamp', () => {
+    before(async function() {
+      await esArchiver.loadIfNeeded('date_nanos_custom');
+      await kibanaServer.uiSettings.replace({ defaultIndex: TEST_INDEX_PATTERN });
+      await kibanaServer.uiSettings.update({
+        'context:defaultSize': `${TEST_DEFAULT_CONTEXT_SIZE}`,
+        'context:step': `${TEST_STEP_SIZE}`,
+      });
+    });
+
+    after(function unloadMakelogs() {
+      return esArchiver.unload('date_nanos_custom');
+    });
+
+    it('displays predessors - anchor - successors in right order ', async function() {
+      await PageObjects.context.navigateTo(TEST_INDEX_PATTERN, '1');
+      const actualRowsText = await docTable.getRowsText();
+      const expectedRowsText = [
+        'Oct 21, 2019 @ 08:30:04.828733000 -',
+        'Oct 21, 2019 @ 00:30:04.828740000 -',
+        'Oct 21, 2019 @ 00:30:04.828723000 -',
+      ];
+      expect(actualRowsText).to.eql(expectedRowsText);
+    });
+  });
+}

test/functional/apps/context/index.js

@@ -42,5 +42,6 @@ export default function({ getService, getPageObjects, loadTestFile }) {
     loadTestFile(require.resolve('./_filters'));
     loadTestFile(require.resolve('./_size'));
     loadTestFile(require.resolve('./_date_nanos'));
+    loadTestFile(require.resolve('./_date_nanos_custom_timestamp'));
   });
 }

test/functional/fixtures/es_archiver/date_nanos_custom/data.json

@@ -0,0 +1,56 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "index-pattern:date_nanos_custom_timestamp",
+    "index": ".kibana",
+    "source": {
+      "index-pattern": {
+        "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"test\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"test.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"test\"}}},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date_nanos\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+        "timeFieldName": "timestamp",
+        "title": "date_nanos_custom_timestamp"
+      },
+      "references": [
+      ],
+      "type": "index-pattern",
+      "updated_at": "2020-01-09T21:43:20.283Z"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 00:30:04.828740"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 08:30:04.828733"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 00:30:04.828723"
+    }
+  }
+}
+
+

test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json

@@ -0,0 +1,31 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": "date_nanos_custom_timestamp",
+    "mappings": {
+      "properties": {
+        "test": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "type": "text"
+        },
+        "timestamp": {
+          "format": "yyyy-MM-dd HH:mm:ss.SSSSSS",
+          "type": "date_nanos"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}