[Stack Monitoring] api tests for cluster and elasticsearch (#145138)

### Summary
Part of #119658

Add api integration tests for cluster and elasticsearch routes to
validate behavior when reading data ingested by elastic-agent.

We currently have a testing suite for legacy and another one for
metricbeat. Since metricbeat and agent documents only differ in their
metadata, for example agent will populate a `data_stream.*` property to
identify the document types while metricbeat uses `metricset.*`, the
tests assertion validating _business_ data should pass regardless of the
documents source. With this in mind the metricbeat tests were updated to
run the tests twice, one time with metricbeat data and a second time
with package data.

To generate the archives the `metrics-*` mappings were extracted with
esArchiver from an elasticsearch with the package installed, and the
documents were transformed from the metricbeat documents with [this
script](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.ts

@@ -6,8 +6,10 @@
  */
 
 import { get, groupBy } from 'lodash';
-import { prefixIndexPatternWithCcs } from '../../../../../common/ccs_utils';
-import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
+import {
+  getIndexPatterns,
+  getElasticsearchDataset,
+} from '../../../../lib/cluster/get_index_patterns';
 import {
   postElasticsearchCcrRequestParamsRT,
   postElasticsearchCcrRequestPayloadRT,
@@ -140,6 +142,13 @@ function buildRequest(
                       },
                     },
                   },
+                  {
+                    term: {
+                      'data_stream.dataset': {
+                        value: getElasticsearchDataset('ccr'),
+                      },
+                    },
+                  },
                 ],
               },
             },
@@ -215,7 +224,14 @@ export function ccrRoute(server: MonitoringCore) {
     async handler(req) {
       const config = server.config;
       const ccs = req.payload.ccs;
-      const esIndexPattern = prefixIndexPatternWithCcs(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+      const dataset = 'ccr';
+      const moduleType = 'elasticsearch';
+      const esIndexPattern = getIndexPatterns({
+        config,
+        moduleType,
+        dataset,
+        ccs,
+      });
 
       try {
         const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');

x-pack/test/api_integration/apis/monitoring/cluster/list_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('list mb', function () {
-    // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
-    this.tags(['skipCloud']);
-
-    describe('with trial license clusters', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster_mb';
-      const timeRange = {
-        min: '2017-08-15T21:00:00Z',
-        max: '2017-08-16T00:00:00Z',
-      };
-      const codePaths = ['all'];
-
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+  describe('list - metricbeat and package', function () {
+    ['mb', 'package'].forEach((source) => {
+      describe(`list ${source}`, function () {
+        // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
+        this.tags(['skipCloud']);
 
-      after('unload clusters archive', () => {
-        return tearDown();
-      });
+        const archive = `x-pack/test/functional/es_archives/monitoring/multicluster_${source}`;
+
+        describe('with trial license clusters', () => {
+          const timeRange = {
+            min: '2017-08-15T21:00:00Z',
+            max: '2017-08-16T00:00:00Z',
+          };
+          const codePaths = ['all'];
+
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
+
+          after('unload clusters archive', () => {
+            return tearDown(archive);
+          });
 
-      it('should load multiple clusters', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, codePaths })
-          .expect(200);
-        expect(body).to.eql(multiclusterFixture);
+          it('should load multiple clusters', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, codePaths })
+              .expect(200);
+            expect(body).to.eql(multiclusterFixture);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/cluster/overview_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('overview mb', function () {
-    // TODO: https://github.com/elastic/stack-monitoring/issues/31
-    this.tags(['skipCloud']);
-
-    describe('with trial license clusters', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_mb';
-      const timeRange = {
-        min: '2017-08-23T21:29:35Z',
-        max: '2017-08-23T21:47:25Z',
-      };
-      const codePaths = ['all'];
-
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+  describe('overview - metricbeat and package', function () {
+    ['mb', 'package'].forEach((source) => {
+      describe(`overview ${source}`, function () {
+        // TODO: https://github.com/elastic/stack-monitoring/issues/31
+        this.tags(['skipCloud']);
 
-      after('unload clusters archive', () => {
-        return tearDown();
-      });
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_${source}`;
+
+        describe('with trial license clusters', () => {
+          const timeRange = {
+            min: '2017-08-23T21:29:35Z',
+            max: '2017-08-23T21:47:25Z',
+          };
+          const codePaths = ['all'];
+
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
+
+          after('unload clusters archive', () => {
+            return tearDown(archive);
+          });
 
-      it('should load multiple clusters', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, codePaths })
-          .expect(200);
-        expect(body).to.eql(overviewFixture);
+          it('should load multiple clusters', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, codePaths })
+              .expect(200);
+            expect(body).to.eql(overviewFixture);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js

@@ -12,32 +12,36 @@ import { getLifecycleMethods } from '../data_stream';
 export default function ({ getService }) {
   const supertest = getService('supertest');
 
-  describe('ccr mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
-    const { setup, tearDown } = getLifecycleMethods(getService);
-    const timeRange = {
-      min: '2018-09-19T00:00:00.000Z',
-      max: '2018-09-19T23:59:59.000Z',
-    };
+  describe('ccr - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`ccr ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
+        const { setup, tearDown } = getLifecycleMethods(getService);
+        const timeRange = {
+          min: '2018-09-19T00:00:00.000Z',
+          max: '2018-09-19T23:59:59.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should return all followers and a grouping of stats by follower index', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-        })
-        .expect(200);
+        it('should return all followers and a grouping of stats by follower index', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+            })
+            .expect(200);
 
-      expect(body).to.eql(ccrFixture);
+          expect(body).to.eql(ccrFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js

@@ -14,35 +14,40 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('ccr shard mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
-    const timeRange = {
-      min: '2018-09-19T00:00:00.000Z',
-      max: '2018-09-19T23:59:59.000Z',
-    };
-
-    before('load archive', () => {
-      return setup(archive);
-    });
+  describe('ccr shard - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+      describe(`ccr shard ${source}`, () => {
+        const timeRange = {
+          min: '2018-09-19T00:00:00.000Z',
+          max: '2018-09-19T23:59:59.000Z',
+        };
+
+        before('load archive', () => {
+          return setup(archive);
+        });
+
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
+
+        it('should return specific shard details', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+            })
+            .expect(200);
 
-    it('should return specific shard details', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-        })
-        .expect(200);
-
-      // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
-      const keysToIgnore = ['stat', 'oldestStat'];
-      expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+          // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
+          const keysToIgnore = ['stat', 'oldestStat'];
+          expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js

@@ -14,50 +14,53 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('index detail mb', () => {
-    const archive =
-      'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
-    const timeRange = {
-      min: '2017-10-05T20:31:48.000Z',
-      max: '2017-10-05T20:35:12.000Z',
-    };
-
-    before('load archive', () => {
-      return setup(archive);
-    });
+  describe('index detail - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`index detail ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+        const timeRange = {
+          min: '2017-10-05T20:31:48.000Z',
+          max: '2017-10-05T20:35:12.000Z',
+        };
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    it('should summarize index with chart metrics data for the non-advanced view', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: false,
-        })
-        .expect(200);
-
-      expect(body).to.eql(indexDetailFixture);
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
+
+        it('should summarize index with chart metrics data for the non-advanced view', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: false,
+            })
+            .expect(200);
+
+          expect(body).to.eql(indexDetailFixture);
+        });
+
+        it('should summarize index with chart metrics data for the advanced view', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: true,
+            })
+            .expect(200);
 
-    it('should summarize index with chart metrics data for the advanced view', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: true,
-        })
-        .expect(200);
-
-      expect(body).to.eql(indexDetailAdvancedFixture);
+          expect(body).to.eql(indexDetailAdvancedFixture);
+        });
+      });
     });
   });
 }