Fix the timestamp field of a data stream to @timestamp

The commit makes the following changes: * The timestamp field of a data stream definition in a composable index template can only be set to '@timestamp'. * Removed custom data stream timestamp field validation and reuse the validation from `TimestampFieldMapper` and instead only check that the _timestamp field mapping has been defined on a backing index of a data stream. * Moved code that injects _timestamp meta field mapping from `MetadataCreateIndexService#applyCreateIndexRequestWithV2Template58956(...)` method to `MetadataIndexTemplateService#collectMappings(...)` method. * Fixed a bug (elastic#58956) that cases timestamp field validation to be performed for each template and instead of the final mappings that is created. Relates to elastic#58642 Relates to elastic#53100 Closes elastic#58956 Closes elastic#58583
martijnvg · Jul 6, 2020 · 5269034 · 5269034
1 parent afb6f9c
commit 5269034
Show file tree

Hide file tree

Showing 18 changed files with 167 additions and 322 deletions.
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml
@@ -10,7 +10,7 @@ setup:
  body:
  index_patterns: logs-*
  data_stream:
- timestamp_field: timestamp
+ timestamp_field: '@timestamp'
  template:
  mappings:
  properties:
@@ -36,7 +36,7 @@ teardown:
  refresh: true
  body:
  foo: bar
- timestamp: '2020-12-12'
+ '@timestamp': '2020-12-12'
 
  - do:
  reindex:
@@ -69,7 +69,7 @@ teardown:
  refresh: true
  body:
  foo: bar
- timestamp: '2020-12-12'
+ '@timestamp': '2020-12-12'
 
  - do:
  reindex:
@@ -102,7 +102,7 @@ teardown:
  refresh: true
  body:
  foo: bar
- timestamp: '2020-12-12'
+ '@timestamp': '2020-12-12'
 
  - do:
  reindex:

diff --git a/...-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml b/...-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml
@@ -51,4 +51,4 @@
  - match: {data_streams.1.name: my_remote_cluster:simple-data-stream2}
  - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
  - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
- - match: {data_streams.1.timestamp_field: "@timestamp2"}
+ - match: {data_streams.1.timestamp_field: "@timestamp"}
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml
@@ -30,7 +30,7 @@
  '@timestamp2':
  type: date
  data_stream:
- timestamp_field: '@timestamp2'
+ timestamp_field: '@timestamp'
 
  - do:
  indices.create_data_stream:

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -25,10 +25,10 @@ setup:
  template:
  mappings:
  properties:
- '@timestamp2':
+ '@timestamp':
  type: date
  data_stream:
- timestamp_field: '@timestamp2'
+ timestamp_field: '@timestamp'
 
 ---
 "Create data stream":
@@ -55,7 +55,7 @@ setup:
  - length: { 0.indices: 1 }
  - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
  - match: { 1.name: simple-data-stream2 }
- - match: { 1.timestamp_field.name: '@timestamp2' }
+ - match: { 1.timestamp_field.name: '@timestamp' }
  - match: { 0.generation: 1 }
  - length: { 1.indices: 1 }
  - match: { 1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
@@ -124,31 +124,26 @@ setup:
  indices.get_data_stream: {}
  - match: { 0.name: simple-data-stream1 }
  - match: { 0.timestamp_field.name: '@timestamp' }
- - match: { 0.timestamp_field.mapping: {type: date} }
  - match: { 0.generation: 1 }
  - match: { 1.name: simple-data-stream2 }
- - match: { 1.timestamp_field.name: '@timestamp2' }
- - match: { 1.timestamp_field.mapping: {type: date} }
+ - match: { 1.timestamp_field.name: '@timestamp' }
  - match: { 1.generation: 1 }
 
  - do:
  indices.get_data_stream:
  name: simple-data-stream1
  - match: { 0.name: simple-data-stream1 }
  - match: { 0.timestamp_field.name: '@timestamp' }
- - match: { 0.timestamp_field.mapping: {type: date} }
  - match: { 0.generation: 1 }
 
  - do:
  indices.get_data_stream:
  name: simple-data-stream*
  - match: { 0.name: simple-data-stream1 }
  - match: { 0.timestamp_field.name: '@timestamp' }
- - match: { 0.timestamp_field.mapping: {type: date} }
  - match: { 0.generation: 1 }
  - match: { 1.name: simple-data-stream2 }
- - match: { 1.timestamp_field.name: '@timestamp2' }
- - match: { 1.timestamp_field.mapping: {type: date} }
+ - match: { 1.timestamp_field.name: '@timestamp' }
  - match: { 1.generation: 1 }
 
  - do:
@@ -235,35 +230,35 @@ setup:
  template:
  mappings:
  properties:
- 'timestamp':
+ '@timestamp':
  type: date
  data_stream:
- timestamp_field: timestamp
+ timestamp_field: '@timestamp'
 
  - do:
  index:
  index: logs-foobar
- body: { timestamp: '2020-12-12' }
+ body: { '@timestamp': '2020-12-12' }
  - match: { _index: .ds-logs-foobar-000001 }
 
  - do:
  catch: bad_request
  index:
  index: .ds-logs-foobar-000001
- body: { timestamp: '2020-12-12' }
+ body: { '@timestamp': '2020-12-12' }
 
  - do:
  bulk:
  body:
  - create:
  _index: .ds-logs-foobar-000001
- - timestamp: '2020-12-12'
+ - '@timestamp': '2020-12-12'
  - index:
  _index: .ds-logs-foobar-000001
- - timestamp: '2020-12-12'
+ - '@timestamp': '2020-12-12'
  - create:
  _index: logs-foobar
- - timestamp: '2020-12-12'
+ - '@timestamp': '2020-12-12'
  - match: { errors: true }
  - match: { items.0.create.status: 400 }
  - match: { items.0.create.error.type: illegal_argument_exception }
@@ -296,10 +291,10 @@ setup:
  template:
  mappings:
  properties:
- 'timestamp':
+ '@timestamp':
  type: date
  data_stream:
- timestamp_field: timestamp
+ timestamp_field: '@timestamp'
 
  - do:
  catch: bad_request
@@ -315,19 +310,19 @@ setup:
  - foo: bar
  - create:
  _index: logs-foobar
- - timestamp: '2020-12-12'
+ - '@timestamp': '2020-12-12'
  - create:
  _index: logs-foobar
- - timestamp: ['2020-12-12', '2022-12-12']
+ - '@timestamp': ['2020-12-12', '2022-12-12']
  - match: { errors: true }
  - match: { items.0.create.status: 400 }
  - match: { items.0.create.error.caused_by.type: illegal_argument_exception }
- - match: { items.0.create.error.caused_by.reason: "data stream timestamp field [timestamp] is missing" }
+ - match: { items.0.create.error.caused_by.reason: "data stream timestamp field [@timestamp] is missing" }
  - match: { items.1.create.result: created }
  - match: { items.1.create._index: .ds-logs-foobar-000001 }
  - match: { items.2.create.status: 400 }
  - match: { items.2.create.error.caused_by.type: illegal_argument_exception }
- - match: { items.2.create.error.caused_by.reason: "data stream timestamp field [timestamp] encountered multiple values" }
+ - match: { items.2.create.error.caused_by.reason: "data stream timestamp field [@timestamp] encountered multiple values" }
 
  - do:
  indices.delete_data_stream:

diff --git a/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml b/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml
@@ -13,22 +13,22 @@
  body:
  index_patterns: logs-*
  data_stream:
- timestamp_field: timestamp
+ timestamp_field: '@timestamp'
  template:
  settings:
  number_of_shards: 1
  number_of_replicas: 0
  mappings:
  properties:
- timestamp:
+ '@timestamp':
  type: date
 
  - do:
  index:
  index: logs-foobar
  refresh: true
  body:
- 'timestamp': '2020-12-12'
+ '@timestamp': '2020-12-12'
  foo: bar
 
  - do:
@@ -43,7 +43,7 @@
  indices.get_data_stream:
  name: logs-foobar
  - match: { 0.name: logs-foobar }
- - match: { 0.timestamp_field.name: 'timestamp' }
+ - match: { 0.timestamp_field.name: '@timestamp' }
  - length: { 0.indices: 1 }
  - match: { 0.indices.0.index_name: '.ds-logs-foobar-000001' }
 

diff --git a/...ec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml b/...ec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
@@ -105,7 +105,7 @@ setup:
  - match: {data_streams.1.name: simple-data-stream2}
  - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
  - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
- - match: {data_streams.1.timestamp_field: "@timestamp2"}
+ - match: {data_streams.1.timestamp_field: "@timestamp"}
 
  - do:
  indices.delete_data_stream:
@@ -165,7 +165,7 @@ setup:
  - match: {data_streams.1.name: simple-data-stream2}
  - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
  - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
- - match: {data_streams.1.timestamp_field: "@timestamp2"}
+ - match: {data_streams.1.timestamp_field: "@timestamp"}
 
  - do:
  indices.delete_data_stream: