diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 927951526b665..e42106afda2c2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1107,6 +1107,7 @@ describe('EPM template', () => { ]); expect(esClient.indices.getDataStream).toBeCalledWith({ name: 'test.*-*', + expand_wildcards: ['open', 'hidden'], }); const putMappingsCall = esClient.indices.putMapping.mock.calls.map(([{ index }]) => index); expect(putMappingsCall).toHaveLength(1); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 4573b5ef7f21b..96e1b5f55ad8a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -605,6 +605,7 @@ const getDataStreams = async ( const body = await esClient.indices.getDataStream({ name: indexTemplate.index_patterns.join(','), + expand_wildcards: ['open', 'hidden'], }); const dataStreams = body.data_streams; diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index d217fed4cc904..46afa9dc0905d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -40,5 +40,6 @@ export default function loadTests({ loadTestFile, getService }) { loadTestFile(require.resolve('./custom_ingest_pipeline')); loadTestFile(require.resolve('./verification_key_id')); loadTestFile(require.resolve('./install_integration_in_multiple_spaces.ts')); + loadTestFile(require.resolve('./install_hidden_datastreams')); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts new file mode 100644 index 0000000000000..c734af44b36f4 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { setupFleetAndAgents } from '../agents/services'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const es = getService('es'); + + const deletePackage = async (name: string, version: string) => { + await supertest.delete(`/api/fleet/epm/packages/${name}/${version}`).set('kbn-xsrf', 'xxxx'); + }; + + describe('installing with hidden datastream', async () => { + skipIfNoDockerRegistry(providerContext); + setupFleetAndAgents(providerContext); + + after(async () => { + await deletePackage('apm', '8.8.0'); + }); + + it('should rollover hidden datastreams when failed to update mappings', async function () { + await supertest + .post(`/api/fleet/epm/packages/apm/8.7.0`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + + await es.index({ + index: 'metrics-apm.service_summary.10m-default', + document: { + '@timestamp': '2023-05-30T07:50:00.000Z', + agent: { + name: 'go', + }, + data_stream: { + dataset: 'apm.service_summary.10m', + namespace: 'default', + type: 'metrics', + }, + ecs: { + version: '8.6.0-dev', + }, + event: { + agent_id_status: 'missing', + ingested: '2023-05-30T07:57:12Z', + }, + metricset: { + interval: '10m', + name: 'service_summary', + }, + observer: { + hostname: '047e282994fb', + type: 'apm-server', + version: '8.7.0', + }, + processor: { + event: 'metric', + name: 'metric', + }, + service: { + language: { + name: 'go', + }, + name: '___main_elastic_cloud_87_ilm_fix', + }, + }, + }); + + await supertest + .post(`/api/fleet/epm/packages/apm/8.8.0`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + + const ds = await es.indices.get({ + index: 'metrics-apm.service_summary*', + expand_wildcards: ['open', 'hidden'], + }); + // datastream rolled over + expect(Object.keys(ds).length).greaterThan(1); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.7.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.7.0.zip new file mode 100644 index 0000000000000..8bffee49e1b6f Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.7.0.zip differ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.8.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.8.0.zip new file mode 100644 index 0000000000000..c3024c351aa8d Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.8.0.zip differ