From 7589eff97aa0194ef6a1f8ea8b7dff5d67d28628 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 4 Aug 2020 10:51:32 +0200 Subject: [PATCH 1/6] [Ingest Manager] Adjust dataset aggs to use datastream fields instead Elastic Agent and Elasticsearch are switching over from using dataset.* to datastream.*. This adjust the aggregation on the dataset page to get the datastreams. For this to work properly, the most recent version of Elasticsearch 7.9 must be used and is pending updates on all the packages to ship also the datastream fields, see https://github.com/elastic/integrations/pull/213 --- .../ingest_manager/server/routes/data_streams/handlers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts index df37aeb27c75c..e22ad81e33981 100644 --- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -54,19 +54,19 @@ export const getListHandler: RequestHandler = async (context, request, response) aggs: { dataset: { terms: { - field: 'dataset.name', + field: 'datastream.dataset', size: 1, }, }, namespace: { terms: { - field: 'dataset.namespace', + field: 'datastream.namespace', size: 1, }, }, type: { terms: { - field: 'dataset.type', + field: 'datastream.type', size: 1, }, }, From 89de0ec67e86036d195b5faf4557d402a1ab1cde Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 4 Aug 2020 13:26:57 -0700 Subject: [PATCH 2/6] Update datastream to data_stream --- .../server/routes/data_streams/handlers.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts index e22ad81e33981..43ae2b72f6077 100644 --- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -31,12 +31,12 @@ export const getListHandler: RequestHandler = async (context, request, response) must: [ { exists: { - field: 'dataset.namespace', + field: 'data_stream.namespace', }, }, { exists: { - field: 'dataset.name', + field: 'data_stream.dataset', }, }, ], @@ -54,19 +54,19 @@ export const getListHandler: RequestHandler = async (context, request, response) aggs: { dataset: { terms: { - field: 'datastream.dataset', + field: 'data_stream.dataset', size: 1, }, }, namespace: { terms: { - field: 'datastream.namespace', + field: 'data_stream.namespace', size: 1, }, }, type: { terms: { - field: 'datastream.type', + field: 'data_stream.type', size: 1, }, }, From 794d3c62b06a4512637fc82c8515ef27bf7b0784 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 4 Aug 2020 13:29:11 -0700 Subject: [PATCH 3/6] Update data stream name generation --- .../epm/elasticsearch/template/template.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index a739806d5868b..cc6bc25a8f8f6 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -393,14 +393,16 @@ const updateExistingIndex = async ({ // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue // to skip updating and assume the value in the index mapping is correct delete mappings.properties.stream; - delete mappings.properties.dataset; + delete mappings.properties.data_stream; - // get the dataset values from the index template to compose data stream name + // get the data_stream values from the index template to compose data stream name const indexMappings = await getIndexMappings(indexName, callCluster); - const dataset = indexMappings[indexName].mappings.properties.dataset.properties; - if (!dataset.type.value || !dataset.name.value || !dataset.namespace.value) - throw new Error(`dataset values are missing from the index template ${indexName}`); - const dataStreamName = `${dataset.type.value}-${dataset.name.value}-${dataset.namespace.value}`; + // Use snake case for easier grepping across files + // eslint-disable-next-line @typescript-eslint/camelcase + const data_stream = indexMappings[indexName].mappings.properties.data_stream.properties; + if (!data_stream.type.value || !data_stream.dataset.value || !data_stream.namespace.value) + throw new Error(`data_stream values are missing from the index template ${indexName}`); + const datastreamName = `${data_stream.type.value}-${data_stream.dataset.value}-${data_stream.namespace.value}`; // try to update the mappings first try { From 0b73367812a837ad812544e72b1b9c621047c48b Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 4 Aug 2020 14:45:31 -0700 Subject: [PATCH 4/6] Fix typo --- .../server/services/epm/elasticsearch/template/template.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index cc6bc25a8f8f6..6ff238d5b48bd 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -413,13 +413,13 @@ const updateExistingIndex = async ({ // if update fails, rollover data stream } catch (err) { try { - const path = `/${dataStreamName}/_rollover`; + const path = `/${datastreamName}/_rollover`; await callCluster('transport.request', { method: 'POST', path, }); } catch (error) { - throw new Error(`cannot rollover data stream ${dataStreamName}`); + throw new Error(`cannot rollover data stream ${datastreamName}`); } } // update settings after mappings was successful to ensure From e3e42a30e8cedcd7533eb372c3f7ee8173015022 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 4 Aug 2020 15:17:19 -0700 Subject: [PATCH 5/6] Temporarily use datastream instead of data_stream --- .../server/routes/data_streams/handlers.ts | 10 +++++----- .../epm/elasticsearch/template/template.ts | 14 ++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts index 43ae2b72f6077..00fa3437e3b11 100644 --- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -31,12 +31,12 @@ export const getListHandler: RequestHandler = async (context, request, response) must: [ { exists: { - field: 'data_stream.namespace', + field: 'datastream.namespace', }, }, { exists: { - field: 'data_stream.dataset', + field: 'datastream.dataset', }, }, ], @@ -54,19 +54,19 @@ export const getListHandler: RequestHandler = async (context, request, response) aggs: { dataset: { terms: { - field: 'data_stream.dataset', + field: 'datastream.dataset', size: 1, }, }, namespace: { terms: { - field: 'data_stream.namespace', + field: 'datastream.namespace', size: 1, }, }, type: { terms: { - field: 'data_stream.type', + field: 'datastream.type', size: 1, }, }, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 6ff238d5b48bd..5d921c52ea713 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -393,16 +393,14 @@ const updateExistingIndex = async ({ // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue // to skip updating and assume the value in the index mapping is correct delete mappings.properties.stream; - delete mappings.properties.data_stream; + delete mappings.properties.datastream; - // get the data_stream values from the index template to compose data stream name + // get the datastream values from the index template to compose data stream name const indexMappings = await getIndexMappings(indexName, callCluster); - // Use snake case for easier grepping across files - // eslint-disable-next-line @typescript-eslint/camelcase - const data_stream = indexMappings[indexName].mappings.properties.data_stream.properties; - if (!data_stream.type.value || !data_stream.dataset.value || !data_stream.namespace.value) - throw new Error(`data_stream values are missing from the index template ${indexName}`); - const datastreamName = `${data_stream.type.value}-${data_stream.dataset.value}-${data_stream.namespace.value}`; + const datastream = indexMappings[indexName].mappings.properties.datastream.properties; + if (!datastream.type.value || !datastream.dataset.value || !datastream.namespace.value) + throw new Error(`datastream values are missing from the index template ${indexName}`); + const datastreamName = `${datastream.type.value}-${datastream.dataset.value}-${datastream.namespace.value}`; // try to update the mappings first try { From 76b94b81857a5ba26b881f9ffbf9c074bb68bb0d Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 5 Aug 2020 10:02:28 +0200 Subject: [PATCH 6/6] updating to use `data_stream` instead of `datastream` --- .../server/routes/data_streams/handlers.ts | 10 +++++----- .../epm/elasticsearch/template/template.ts | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts index 00fa3437e3b11..43ae2b72f6077 100644 --- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -31,12 +31,12 @@ export const getListHandler: RequestHandler = async (context, request, response) must: [ { exists: { - field: 'datastream.namespace', + field: 'data_stream.namespace', }, }, { exists: { - field: 'datastream.dataset', + field: 'data_stream.dataset', }, }, ], @@ -54,19 +54,19 @@ export const getListHandler: RequestHandler = async (context, request, response) aggs: { dataset: { terms: { - field: 'datastream.dataset', + field: 'data_stream.dataset', size: 1, }, }, namespace: { terms: { - field: 'datastream.namespace', + field: 'data_stream.namespace', size: 1, }, }, type: { terms: { - field: 'datastream.type', + field: 'data_stream.type', size: 1, }, }, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 5d921c52ea713..71e49acf1766f 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -393,14 +393,14 @@ const updateExistingIndex = async ({ // are added in https://github.com/elastic/kibana/issues/66551. namespace value we will continue // to skip updating and assume the value in the index mapping is correct delete mappings.properties.stream; - delete mappings.properties.datastream; + delete mappings.properties.data_stream; - // get the datastream values from the index template to compose data stream name + // get the data_stream values from the index template to compose data stream name const indexMappings = await getIndexMappings(indexName, callCluster); - const datastream = indexMappings[indexName].mappings.properties.datastream.properties; - if (!datastream.type.value || !datastream.dataset.value || !datastream.namespace.value) - throw new Error(`datastream values are missing from the index template ${indexName}`); - const datastreamName = `${datastream.type.value}-${datastream.dataset.value}-${datastream.namespace.value}`; + const dataStream = indexMappings[indexName].mappings.properties.data_stream.properties; + if (!dataStream.type.value || !dataStream.dataset.value || !dataStream.namespace.value) + throw new Error(`data_stream values are missing from the index template ${indexName}`); + const dataStreamName = `${dataStream.type.value}-${dataStream.dataset.value}-${dataStream.namespace.value}`; // try to update the mappings first try { @@ -411,13 +411,13 @@ const updateExistingIndex = async ({ // if update fails, rollover data stream } catch (err) { try { - const path = `/${datastreamName}/_rollover`; + const path = `/${dataStreamName}/_rollover`; await callCluster('transport.request', { method: 'POST', path, }); } catch (error) { - throw new Error(`cannot rollover data stream ${datastreamName}`); + throw new Error(`cannot rollover data stream ${dataStreamName}`); } } // update settings after mappings was successful to ensure