[Stack Monitoring] api tests for cluster and elasticsearch (elastic#1…

…45138)

### Summary
Part of elastic#119658

Add api integration tests for cluster and elasticsearch routes to
validate behavior when reading data ingested by elastic-agent.

We currently have a testing suite for legacy and another one for
metricbeat. Since metricbeat and agent documents only differ in their
metadata, for example agent will populate a `data_stream.*` property to
identify the document types while metricbeat uses `metricset.*`, the
tests assertion validating _business_ data should pass regardless of the
documents source. With this in mind the metricbeat tests were updated to
run the tests twice, one time with metricbeat data and a second time
with package data.

To generate the archives the `metrics-*` mappings were extracted with
esArchiver from an elasticsearch with the package installed, and the
documents were transformed from the metricbeat documents with [this
script](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit 5cf0d0f)

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.ts

@@ -6,8 +6,10 @@
  */
 
 import { get, groupBy } from 'lodash';
-import { prefixIndexPatternWithCcs } from '../../../../../common/ccs_utils';
-import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
+import {
+ getIndexPatterns,
+ getElasticsearchDataset,
+} from '../../../../lib/cluster/get_index_patterns';
 import {
  postElasticsearchCcrRequestParamsRT,
  postElasticsearchCcrRequestPayloadRT,
@@ -140,6 +142,13 @@ function buildRequest(
  },
  },
  },
+ {
+ term: {
+ 'data_stream.dataset': {
+ value: getElasticsearchDataset('ccr'),
+ },
+ },
+ },
  ],
  },
  },
@@ -215,7 +224,14 @@ export function ccrRoute(server: MonitoringCore) {
  async handler(req) {
  const config = server.config;
  const ccs = req.payload.ccs;
- const esIndexPattern = prefixIndexPatternWithCcs(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+ const dataset = 'ccr';
+ const moduleType = 'elasticsearch';
+ const esIndexPattern = getIndexPatterns({
+ config,
+ moduleType,
+ dataset,
+ ccs,
+ });
 
  try {
  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');

x-pack/test/api_integration/apis/monitoring/cluster/list_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('list mb', function () {
- // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
- this.tags(['skipCloud']);
-
- describe('with trial license clusters', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster_mb';
- const timeRange = {
- min: '2017-08-15T21:00:00Z',
- max: '2017-08-16T00:00:00Z',
- };
- const codePaths = ['all'];
-
- before('load clusters archive', () => {
- return setup(archive);
- });
+ describe('list - metricbeat and package', function () {
+ ['mb', 'package'].forEach((source) => {
+ describe(`list ${source}`, function () {
+ // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
+ this.tags(['skipCloud']);
 
- after('unload clusters archive', () => {
- return tearDown();
- });
+ const archive = `x-pack/test/functional/es_archives/monitoring/multicluster_${source}`;
+
+ describe('with trial license clusters', () => {
+ const timeRange = {
+ min: '2017-08-15T21:00:00Z',
+ max: '2017-08-16T00:00:00Z',
+ };
+ const codePaths = ['all'];
+
+ before('load clusters archive', () => {
+ return setup(archive);
+ });
+
+ after('unload clusters archive', () => {
+ return tearDown(archive);
+ });
 
- it('should load multiple clusters', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters')
- .set('kbn-xsrf', 'xxx')
- .send({ timeRange, codePaths })
- .expect(200);
- expect(body).to.eql(multiclusterFixture);
+ it('should load multiple clusters', async () => {
+ const { body } = await supertest
+ .post('/api/monitoring/v1/clusters')
+ .set('kbn-xsrf', 'xxx')
+ .send({ timeRange, codePaths })
+ .expect(200);
+ expect(body).to.eql(multiclusterFixture);
+ });
+ });
  });
  });
  });

x-pack/test/api_integration/apis/monitoring/cluster/overview_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('overview mb', function () {
- // TODO: https://github.com/elastic/stack-monitoring/issues/31
- this.tags(['skipCloud']);
-
- describe('with trial license clusters', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_mb';
- const timeRange = {
- min: '2017-08-23T21:29:35Z',
- max: '2017-08-23T21:47:25Z',
- };
- const codePaths = ['all'];
-
- before('load clusters archive', () => {
- return setup(archive);
- });
+ describe('overview - metricbeat and package', function () {
+ ['mb', 'package'].forEach((source) => {
+ describe(`overview ${source}`, function () {
+ // TODO: https://github.com/elastic/stack-monitoring/issues/31
+ this.tags(['skipCloud']);
 
- after('unload clusters archive', () => {
- return tearDown();
- });
+ const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_${source}`;
+
+ describe('with trial license clusters', () => {
+ const timeRange = {
+ min: '2017-08-23T21:29:35Z',
+ max: '2017-08-23T21:47:25Z',
+ };
+ const codePaths = ['all'];
+
+ before('load clusters archive', () => {
+ return setup(archive);
+ });
+
+ after('unload clusters archive', () => {
+ return tearDown(archive);
+ });
 
- it('should load multiple clusters', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
- .set('kbn-xsrf', 'xxx')
- .send({ timeRange, codePaths })
- .expect(200);
- expect(body).to.eql(overviewFixture);
+ it('should load multiple clusters', async () => {
+ const { body } = await supertest
+ .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
+ .set('kbn-xsrf', 'xxx')
+ .send({ timeRange, codePaths })
+ .expect(200);
+ expect(body).to.eql(overviewFixture);
+ });
+ });
  });
  });
  });

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js

@@ -12,32 +12,36 @@ import { getLifecycleMethods } from '../data_stream';
 export default function ({ getService }) {
  const supertest = getService('supertest');
 
- describe('ccr mb', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
- const { setup, tearDown } = getLifecycleMethods(getService);
- const timeRange = {
- min: '2018-09-19T00:00:00.000Z',
- max: '2018-09-19T23:59:59.000Z',
- };
+ describe('ccr - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ describe(`ccr ${source}`, () => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
+ const { setup, tearDown } = getLifecycleMethods(getService);
+ const timeRange = {
+ min: '2018-09-19T00:00:00.000Z',
+ max: '2018-09-19T23:59:59.000Z',
+ };
 
- before('load archive', () => {
- return setup(archive);
- });
+  before('load archive', () => {
+  return setup(archive);
+  });
 
- after('unload archive', () => {
- return tearDown();
- });
+  after('unload archive', () => {
+  return tearDown(archive);
+  });
 
- it('should return all followers and a grouping of stats by follower index', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- })
- .expect(200);
+  it('should return all followers and a grouping of stats by follower index', async () => {
+  const { body } = await supertest
+  .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
+  .set('kbn-xsrf', 'xxx')
+  .send({
+  timeRange,
+  })
+  .expect(200);
 
- expect(body).to.eql(ccrFixture);
+ expect(body).to.eql(ccrFixture);
+ });
+ });
  });
  });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js

@@ -14,35 +14,40 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('ccr shard mb', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
- const timeRange = {
- min: '2018-09-19T00:00:00.000Z',
- max: '2018-09-19T23:59:59.000Z',
- };
-
- before('load archive', () => {
- return setup(archive);
- });
+ describe('ccr shard - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
 
- after('unload archive', () => {
- return tearDown();
- });
+ describe(`ccr shard ${source}`, () => {
+ const timeRange = {
+ min: '2018-09-19T00:00:00.000Z',
+ max: '2018-09-19T23:59:59.000Z',
+ };
+
+ before('load archive', () => {
+ return setup(archive);
+ });
+
+ after('unload archive', () => {
+ return tearDown(archive);
+ });
+
+ it('should return specific shard details', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ })
+ .expect(200);
 
- it('should return specific shard details', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- })
- .expect(200);
-
- // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
- const keysToIgnore = ['stat', 'oldestStat'];
- expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+ // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
+ const keysToIgnore = ['stat', 'oldestStat'];
+ expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+ });
+ });
  });
  });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js

@@ -14,50 +14,53 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('index detail mb', () => {
- const archive =
- 'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
- const timeRange = {
- min: '2017-10-05T20:31:48.000Z',
- max: '2017-10-05T20:35:12.000Z',
- };
-
- before('load archive', () => {
- return setup(archive);
- });
+ describe('index detail - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ describe(`index detail ${source}`, () => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+ const timeRange = {
+ min: '2017-10-05T20:31:48.000Z',
+ max: '2017-10-05T20:35:12.000Z',
+ };
 
- after('unload archive', () => {
- return tearDown();
- });
+  before('load archive', () => {
+  return setup(archive);
+  });
 
- it('should summarize index with chart metrics data for the non-advanced view', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- is_advanced: false,
- })
- .expect(200);
-
- expect(body).to.eql(indexDetailFixture);
- });
+ after('unload archive', () => {
+ return tearDown(archive);
+ });
+
+ it('should summarize index with chart metrics data for the non-advanced view', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ is_advanced: false,
+ })
+ .expect(200);
+
+ expect(body).to.eql(indexDetailFixture);
+ });
+
+ it('should summarize index with chart metrics data for the advanced view', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ is_advanced: true,
+ })
+ .expect(200);
 
- it('should summarize index with chart metrics data for the advanced view', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- is_advanced: true,
- })
- .expect(200);
-
- expect(body).to.eql(indexDetailAdvancedFixture);
+ expect(body).to.eql(indexDetailAdvancedFixture);
+ });
+ });
  });
  });
 }