From 1ad41db0d982a3c73d6e4f2e3775b434071dc63f Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 25 Jan 2024 09:07:43 -0500 Subject: [PATCH] [Fleet] Update Fleet's custom ingest pipeline names to avoid collisions + add descriptions to each pipeline (#175448) ## Summary Closes https://github.com/elastic/kibana/issues/175254 Ref https://github.com/elastic/kibana/issues/168019 Ref https://github.com/elastic/kibana/pull/170270 In 8.12.0, Fleet unintentionally shipped a breaking change in https://github.com/elastic/kibana/pull/170270 for APM users who make use of a custom `traces-apm` data stream. If a user had previously defined this ingest pipeline to customize documents ingested for the `traces-apm` data stream (defined [here](https://github.com/elastic/integrations/blob/9a36183f0bd12e39a957d2f7bd65f3de4ee685b1/packages/apm/data_stream/traces/manifest.yml#L2-L3), then they would unexpectedly see that pipeline called when documents were ingested to the `traces-apm.rum` and `traces-apm.sampled` datastreams as well. This PR addresses this collision by adding a `.package` suffix to the "package level" ingest pipeline introduced in 8.12.0. So, in 8.12.0 a processor would be defined as such on the `traces-apm.rum` or `traces-apm.sampled` ingest pipeline ``` { "pipeline": { "name": "traces-apm@custom", "ignore_missing_pipeline": true, } }, ``` This PR replaces the pipeline with one that looks as follows: ``` { "pipeline": { "name": "traces-apm.package@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Pipeline for all data streams of type `traces` defined by the `apm` integration" } }, ``` **To be clear: this is a breaking change if you have defined the `traces-apm@custom` integration on 8.12. In 8.12.1, it will no longer be called for documents ingested to the `traces-apm`, `traces-apm.rum`, or `traces-apm.sampled` data streams. You will need to rename your pipeline to `traces-apm.package@custom` to preserve this behavior.** This change also applies to `logs-elastic_agent.*` ingest pipelines. See [this comment](https://github.com/elastic/kibana/issues/175254#issuecomment-1906202137) for more information. There is still technically room for a collision, though it's unlikely, if the data stream name is `package`. This will be handled by a package spec validation proposed in https://github.com/elastic/package-spec/issues/699. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 9fe5a66faf4e06fc444c6078edafc29e91126f8d) --- .../elasticsearch/ingest_pipeline/helpers.test.ts | 7 +++++-- .../epm/elasticsearch/ingest_pipeline/helpers.ts | 8 +++++++- .../apis/epm/custom_ingest_pipeline.ts | 13 +++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts index ebffc0064b989..d759dec042247 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts @@ -188,7 +188,7 @@ processors: }); expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot( - `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true}}]}"` + `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test\` dataset\\"}}]}"` ); }); @@ -231,12 +231,15 @@ processors: - pipeline: name: global@custom ignore_missing_pipeline: true + description: '[Fleet] Global pipeline for all data streams' - pipeline: name: logs@custom ignore_missing_pipeline: true + description: '[Fleet] Pipeline for all data streams of type \`logs\`' - pipeline: name: logs-test.access@custom ignore_missing_pipeline: true + description: '[Fleet] Pipeline for the \`test.access\` dataset' - reroute: tag: test.access dataset: test.reroute @@ -280,7 +283,7 @@ processors: }); expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot( - `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"` + `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test.access\` dataset\\"}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"` ); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts index 9c244821fd013..5ab17d690616e 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts @@ -80,25 +80,30 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor( pipeline.dataStream?.routing_rules?.find( (rule) => rule.source_dataset === pipeline.dataStream?.dataset )?.rules ?? []; + const customPipelineProcessors = [ { pipeline: { name: 'global@custom', ignore_missing_pipeline: true, + description: '[Fleet] Global pipeline for all data streams', }, }, { pipeline: { name: `${pipeline.dataStream.type}@custom`, ignore_missing_pipeline: true, + description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\``, }, }, ...(pipeline.dataStream.package ? [ { pipeline: { - name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}@custom`, + // This pipeline name gets the `.integration` suffix to avoid conflicts with the pipeline name for the dataset below + name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}.integration@custom`, ignore_missing_pipeline: true, + description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\` defined by the \`${pipeline.dataStream.package}\` integration`, }, }, ] @@ -107,6 +112,7 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor( pipeline: { name: `${pipeline.dataStream.type}-${pipeline.dataStream.dataset}@custom`, ignore_missing_pipeline: true, + description: `[Fleet] Pipeline for the \`${pipeline.dataStream.dataset}\` dataset`, }, }, ]; diff --git a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts index 244bc0c9db6d7..6c93e3e6eb14d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts @@ -105,12 +105,12 @@ export default function (providerContext: FtrProviderContext) { }); await es.ingest.putPipeline({ - id: `logs-log@custom`, + id: `logs-log.integration@custom`, processors: [ { append: { field: 'test', - value: ['logs-log'], + value: ['logs-log.integration'], }, }, ], @@ -138,7 +138,7 @@ export default function (providerContext: FtrProviderContext) { id: 'logs@custom', }), es.ingest.deletePipeline({ - id: 'logs-log@custom', + id: 'logs-log.integration@custom', }), es.ingest.deletePipeline({ id: CUSTOM_PIPELINE, @@ -158,7 +158,12 @@ export default function (providerContext: FtrProviderContext) { id: res._id, index: res._index, }); - expect(doc._source?.test).be.eql(['global', 'logs', 'logs-log', 'logs-log.log']); + expect(doc._source?.test).be.eql([ + 'global', + 'logs', + 'logs-log.integration', + 'logs-log.log', + ]); }); }); });