Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[8.12] [Fleet] Update Fleet's custom ingest pipeline names to avoid collisions + add descriptions to each pipeline (#175448) #175547

Merged
merged 1 commit into from
Jan 25, 2024
Merged

[8.12] [Fleet] Update Fleet's custom ingest pipeline names to avoid collisions + add descriptions to each pipeline (#175448) #175547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ processors:
});

expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot(
`"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true}}]}"`
`"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test\` dataset\\"}}]}"`
);
});

Expand Down Expand Up @@ -231,12 +231,15 @@ processors:
- pipeline:
name: global@custom
ignore_missing_pipeline: true
description: '[Fleet] Global pipeline for all data streams'
- pipeline:
name: logs@custom
ignore_missing_pipeline: true
description: '[Fleet] Pipeline for all data streams of type \`logs\`'
- pipeline:
name: logs-test.access@custom
ignore_missing_pipeline: true
description: '[Fleet] Pipeline for the \`test.access\` dataset'
- reroute:
tag: test.access
dataset: test.reroute
Expand Down Expand Up @@ -280,7 +283,7 @@ processors:
});

expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot(
`"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"`
`"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test.access\` dataset\\"}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"`
);
});
});
Expand Down
8 changes: 7 additions & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,25 +80,30 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor(
pipeline.dataStream?.routing_rules?.find(
(rule) => rule.source_dataset === pipeline.dataStream?.dataset
)?.rules ?? [];

const customPipelineProcessors = [
{
pipeline: {
name: 'global@custom',
ignore_missing_pipeline: true,
description: '[Fleet] Global pipeline for all data streams',
},
},
{
pipeline: {
name: `${pipeline.dataStream.type}@custom`,
ignore_missing_pipeline: true,
description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\``,
},
},
...(pipeline.dataStream.package
? [
{
pipeline: {
name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}@custom`,
// This pipeline name gets the `.integration` suffix to avoid conflicts with the pipeline name for the dataset below
name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}.integration@custom`,
ignore_missing_pipeline: true,
description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\` defined by the \`${pipeline.dataStream.package}\` integration`,
},
},
]
Expand All @@ -107,6 +112,7 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor(
pipeline: {
name: `${pipeline.dataStream.type}-${pipeline.dataStream.dataset}@custom`,
ignore_missing_pipeline: true,
description: `[Fleet] Pipeline for the \`${pipeline.dataStream.dataset}\` dataset`,
},
},
];
Expand Down
13 changes: 9 additions & 4 deletions x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,12 @@ export default function (providerContext: FtrProviderContext) {
});

await es.ingest.putPipeline({
id: `logs-log@custom`,
id: `logs-log.integration@custom`,
processors: [
{
append: {
field: 'test',
value: ['logs-log'],
value: ['logs-log.integration'],
},
},
],
Expand Down Expand Up @@ -138,7 +138,7 @@ export default function (providerContext: FtrProviderContext) {
id: 'logs@custom',
}),
es.ingest.deletePipeline({
id: 'logs-log@custom',
id: 'logs-log.integration@custom',
}),
es.ingest.deletePipeline({
id: CUSTOM_PIPELINE,
Expand All @@ -158,7 +158,12 @@ export default function (providerContext: FtrProviderContext) {
id: res._id,
index: res._index,
});
expect(doc._source?.test).be.eql(['global', 'logs', 'logs-log', 'logs-log.log']);
expect(doc._source?.test).be.eql([
'global',
'logs',
'logs-log.integration',
'logs-log.log',
]);
});
});
});
Expand Down
Loading