[8.8] [Fleet] Include hidden data streams in package upgrade (#158654) (

#158698) # Backport This will backport the following commits from `main` to `8.8`: - [[Fleet] Include hidden data streams in package upgrade (#158654)](#158654)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  --------- Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
elastic · May 31, 2023 · cf1a516 · cf1a516
1 parent 9240a6e
commit cf1a516
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 0 deletions.
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -1080,6 +1080,7 @@ describe('EPM template', () => {
       ]);
       expect(esClient.indices.getDataStream).toBeCalledWith({
         name: 'test.*-*',
+        expand_wildcards: ['open', 'hidden'],
       });
       const putMappingsCall = esClient.indices.putMapping.mock.calls.map(([{ index }]) => index);
       expect(putMappingsCall).toHaveLength(1);

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -600,6 +600,7 @@ const getDataStreams = async (
 
   const body = await esClient.indices.getDataStream({
     name: indexTemplate.index_patterns.join(','),
+    expand_wildcards: ['open', 'hidden'],
   });
 
   const dataStreams = body.data_streams;

diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js
@@ -40,5 +40,6 @@ export default function loadTests({ loadTestFile, getService }) {
     loadTestFile(require.resolve('./custom_ingest_pipeline'));
     loadTestFile(require.resolve('./verification_key_id'));
     loadTestFile(require.resolve('./install_integration_in_multiple_spaces.ts'));
+    loadTestFile(require.resolve('./install_hidden_datastreams'));
   });
 }
diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+import { setupFleetAndAgents } from '../agents/services';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const es = getService('es');
+
+  const deletePackage = async (name: string, version: string) => {
+    await supertest.delete(`/api/fleet/epm/packages/${name}/${version}`).set('kbn-xsrf', 'xxxx');
+  };
+
+  describe('installing with hidden datastream', async () => {
+    skipIfNoDockerRegistry(providerContext);
+    setupFleetAndAgents(providerContext);
+
+    after(async () => {
+      await deletePackage('apm', '8.8.0');
+    });
+
+    it('should rollover hidden datastreams when failed to update mappings', async function () {
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.7.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      await es.index({
+        index: 'metrics-apm.service_summary.10m-default',
+        document: {
+          '@timestamp': '2023-05-30T07:50:00.000Z',
+          agent: {
+            name: 'go',
+          },
+          data_stream: {
+            dataset: 'apm.service_summary.10m',
+            namespace: 'default',
+            type: 'metrics',
+          },
+          ecs: {
+            version: '8.6.0-dev',
+          },
+          event: {
+            agent_id_status: 'missing',
+            ingested: '2023-05-30T07:57:12Z',
+          },
+          metricset: {
+            interval: '10m',
+            name: 'service_summary',
+          },
+          observer: {
+            hostname: '047e282994fb',
+            type: 'apm-server',
+            version: '8.7.0',
+          },
+          processor: {
+            event: 'metric',
+            name: 'metric',
+          },
+          service: {
+            language: {
+              name: 'go',
+            },
+            name: '___main_elastic_cloud_87_ilm_fix',
+          },
+        },
+      });
+
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.8.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      const ds = await es.indices.get({
+        index: 'metrics-apm.service_summary*',
+        expand_wildcards: ['open', 'hidden'],
+      });
+      // datastream rolled over
+      expect(Object.keys(ds).length).greaterThan(1);
+    });
+  });
+}
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.7.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.7.0.zip
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.8.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/apm/apm-8.8.0.zip