[Fleet] Include hidden data streams in package upgrade

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -1107,6 +1107,7 @@ describe('EPM template', () => {
       ]);
       expect(esClient.indices.getDataStream).toBeCalledWith({
         name: 'test.*-*',
+        expand_wildcards: ['open', 'hidden'],
       });
       const putMappingsCall = esClient.indices.putMapping.mock.calls.map(([{ index }]) => index);
       expect(putMappingsCall).toHaveLength(1);

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -605,6 +605,7 @@ const getDataStreams = async (
 
   const body = await esClient.indices.getDataStream({
     name: indexTemplate.index_patterns.join(','),
+    expand_wildcards: ['open', 'hidden'],
   });
 
   const dataStreams = body.data_streams;

x-pack/test/fleet_api_integration/apis/epm/index.js

@@ -40,5 +40,6 @@ export default function loadTests({ loadTestFile, getService }) {
     loadTestFile(require.resolve('./custom_ingest_pipeline'));
     loadTestFile(require.resolve('./verification_key_id'));
     loadTestFile(require.resolve('./install_integration_in_multiple_spaces.ts'));
+    loadTestFile(require.resolve('./install_hidden_datastreams'));
   });
 }

x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+import { setupFleetAndAgents } from '../agents/services';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const es = getService('es');
+
+  const deletePackage = async (name: string, version: string) => {
+    await supertest.delete(`/api/fleet/epm/packages/${name}/${version}`).set('kbn-xsrf', 'xxxx');
+  };
+
+  describe('installing with hidden datastream', async () => {
+    skipIfNoDockerRegistry(providerContext);
+    setupFleetAndAgents(providerContext);
+
+    after(async () => {
+      await deletePackage('apm', '8.8.0');
+    });
+
+    it('should rollover hidden datastreams when failed to update mappings', async function () {
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.7.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      await es.index({
+        index: 'metrics-apm.service_summary.10m-default',
+        document: {
+          '@timestamp': '2023-05-30T07:50:00.000Z',
+          agent: {
+            name: 'go',
+          },
+          data_stream: {
+            dataset: 'apm.service_summary.10m',
+            namespace: 'default',
+            type: 'metrics',
+          },
+          ecs: {
+            version: '8.6.0-dev',
+          },
+          event: {
+            agent_id_status: 'missing',
+            ingested: '2023-05-30T07:57:12Z',
+          },
+          metricset: {
+            interval: '10m',
+            name: 'service_summary',
+          },
+          observer: {
+            hostname: '047e282994fb',
+            type: 'apm-server',
+            version: '8.7.0',
+          },
+          processor: {
+            event: 'metric',
+            name: 'metric',
+          },
+          service: {
+            language: {
+              name: 'go',
+            },
+            name: '___main_elastic_cloud_87_ilm_fix',
+          },
+        },
+      });
+
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.8.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      const ds = await es.indices.get({
+        index: 'metrics-apm.service_summary*',
+        expand_wildcards: ['open', 'hidden'],
+      });
+      // datastream rolled over
+      expect(Object.keys(ds).length).greaterThan(1);
+    });
+  });
+}