[8.6] [Stack Monitoring] api tests for cluster and elasticsearch (#14…

…5138) (#145985)

# Backport

This will backport the following commits from `main` to `8.6`:
- [[Stack Monitoring] api tests for cluster and elasticsearch
(#145138)](#145138)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kevin
Lacabane","email":"kevin.lacabane@elastic.co"},"sourceCommit":{"committedDate":"2022-11-22T13:02:50Z","message":"[Stack
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
#119658 api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Infra
Monitoring UI","release_note:skip","Feature:Stack
Monitoring","v8.6.0","v8.7.0"],"number":145138,"url":"#145138
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
#119658 api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296"}},"sourceBranch":"main","suggestedTargetBranches":["8.6"],"targetPullRequestStates":[{"branch":"8.6","label":"v8.6.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"#145138
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
#119658 api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296"}}]}]
BACKPORT-->

Co-authored-by: Kevin Lacabane <kevin.lacabane@elastic.co>

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.ts

@@ -6,8 +6,10 @@
  */
 
 import { get, groupBy } from 'lodash';
-import { prefixIndexPatternWithCcs } from '../../../../../common/ccs_utils';
-import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
+import {
+ getIndexPatterns,
+ getElasticsearchDataset,
+} from '../../../../lib/cluster/get_index_patterns';
 import {
  postElasticsearchCcrRequestParamsRT,
  postElasticsearchCcrRequestPayloadRT,
@@ -140,6 +142,13 @@ function buildRequest(
  },
  },
  },
+ {
+ term: {
+ 'data_stream.dataset': {
+ value: getElasticsearchDataset('ccr'),
+ },
+ },
+ },
  ],
  },
  },
@@ -215,7 +224,14 @@ export function ccrRoute(server: MonitoringCore) {
  async handler(req) {
  const config = server.config;
  const ccs = req.payload.ccs;
- const esIndexPattern = prefixIndexPatternWithCcs(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+ const dataset = 'ccr';
+ const moduleType = 'elasticsearch';
+ const esIndexPattern = getIndexPatterns({
+ config,
+ moduleType,
+ dataset,
+ ccs,
+ });
 
  try {
  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');

x-pack/test/api_integration/apis/monitoring/cluster/list_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('list mb', function () {
- // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
- this.tags(['skipCloud']);
-
- describe('with trial license clusters', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster_mb';
- const timeRange = {
- min: '2017-08-15T21:00:00Z',
- max: '2017-08-16T00:00:00Z',
- };
- const codePaths = ['all'];
-
- before('load clusters archive', () => {
- return setup(archive);
- });
+ describe('list - metricbeat and package', function () {
+ ['mb', 'package'].forEach((source) => {
+ describe(`list ${source}`, function () {
+ // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
+ this.tags(['skipCloud']);
 
- after('unload clusters archive', () => {
- return tearDown();
- });
+ const archive = `x-pack/test/functional/es_archives/monitoring/multicluster_${source}`;
+
+ describe('with trial license clusters', () => {
+ const timeRange = {
+ min: '2017-08-15T21:00:00Z',
+ max: '2017-08-16T00:00:00Z',
+ };
+ const codePaths = ['all'];
+
+ before('load clusters archive', () => {
+ return setup(archive);
+ });
+
+ after('unload clusters archive', () => {
+ return tearDown(archive);
+ });
 
- it('should load multiple clusters', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters')
- .set('kbn-xsrf', 'xxx')
- .send({ timeRange, codePaths })
- .expect(200);
- expect(body).to.eql(multiclusterFixture);
+ it('should load multiple clusters', async () => {
+ const { body } = await supertest
+ .post('/api/monitoring/v1/clusters')
+ .set('kbn-xsrf', 'xxx')
+ .send({ timeRange, codePaths })
+ .expect(200);
+ expect(body).to.eql(multiclusterFixture);
+ });
+ });
  });
  });
  });

x-pack/test/api_integration/apis/monitoring/cluster/overview_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('overview mb', function () {
- // TODO: https://github.com/elastic/stack-monitoring/issues/31
- this.tags(['skipCloud']);
-
- describe('with trial license clusters', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_mb';
- const timeRange = {
- min: '2017-08-23T21:29:35Z',
- max: '2017-08-23T21:47:25Z',
- };
- const codePaths = ['all'];
-
- before('load clusters archive', () => {
- return setup(archive);
- });
+ describe('overview - metricbeat and package', function () {
+ ['mb', 'package'].forEach((source) => {
+ describe(`overview ${source}`, function () {
+ // TODO: https://github.com/elastic/stack-monitoring/issues/31
+ this.tags(['skipCloud']);
 
- after('unload clusters archive', () => {
- return tearDown();
- });
+ const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_${source}`;
+
+ describe('with trial license clusters', () => {
+ const timeRange = {
+ min: '2017-08-23T21:29:35Z',
+ max: '2017-08-23T21:47:25Z',
+ };
+ const codePaths = ['all'];
+
+ before('load clusters archive', () => {
+ return setup(archive);
+ });
+
+ after('unload clusters archive', () => {
+ return tearDown(archive);
+ });
 
- it('should load multiple clusters', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
- .set('kbn-xsrf', 'xxx')
- .send({ timeRange, codePaths })
- .expect(200);
- expect(body).to.eql(overviewFixture);
+ it('should load multiple clusters', async () => {
+ const { body } = await supertest
+ .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
+ .set('kbn-xsrf', 'xxx')
+ .send({ timeRange, codePaths })
+ .expect(200);
+ expect(body).to.eql(overviewFixture);
+ });
+ });
  });
  });
  });

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js

@@ -12,32 +12,36 @@ import { getLifecycleMethods } from '../data_stream';
 export default function ({ getService }) {
  const supertest = getService('supertest');
 
- describe('ccr mb', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
- const { setup, tearDown } = getLifecycleMethods(getService);
- const timeRange = {
- min: '2018-09-19T00:00:00.000Z',
- max: '2018-09-19T23:59:59.000Z',
- };
+ describe('ccr - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ describe(`ccr ${source}`, () => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
+ const { setup, tearDown } = getLifecycleMethods(getService);
+ const timeRange = {
+ min: '2018-09-19T00:00:00.000Z',
+ max: '2018-09-19T23:59:59.000Z',
+ };
 
- before('load archive', () => {
- return setup(archive);
- });
+  before('load archive', () => {
+  return setup(archive);
+  });
 
- after('unload archive', () => {
- return tearDown();
- });
+  after('unload archive', () => {
+  return tearDown(archive);
+  });
 
- it('should return all followers and a grouping of stats by follower index', async () => {
- const { body } = await supertest
- .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- })
- .expect(200);
+  it('should return all followers and a grouping of stats by follower index', async () => {
+  const { body } = await supertest
+  .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
+  .set('kbn-xsrf', 'xxx')
+  .send({
+  timeRange,
+  })
+  .expect(200);
 
- expect(body).to.eql(ccrFixture);
+ expect(body).to.eql(ccrFixture);
+ });
+ });
  });
  });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js

@@ -14,35 +14,40 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('ccr shard mb', () => {
- const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
- const timeRange = {
- min: '2018-09-19T00:00:00.000Z',
- max: '2018-09-19T23:59:59.000Z',
- };
-
- before('load archive', () => {
- return setup(archive);
- });
+ describe('ccr shard - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
 
- after('unload archive', () => {
- return tearDown();
- });
+ describe(`ccr shard ${source}`, () => {
+ const timeRange = {
+ min: '2018-09-19T00:00:00.000Z',
+ max: '2018-09-19T23:59:59.000Z',
+ };
+
+ before('load archive', () => {
+ return setup(archive);
+ });
+
+ after('unload archive', () => {
+ return tearDown(archive);
+ });
+
+ it('should return specific shard details', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ })
+ .expect(200);
 
- it('should return specific shard details', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- })
- .expect(200);
-
- // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
- const keysToIgnore = ['stat', 'oldestStat'];
- expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+ // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
+ const keysToIgnore = ['stat', 'oldestStat'];
+ expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+ });
+ });
  });
  });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js

@@ -14,50 +14,53 @@ export default function ({ getService }) {
  const supertest = getService('supertest');
  const { setup, tearDown } = getLifecycleMethods(getService);
 
- describe('index detail mb', () => {
- const archive =
- 'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
- const timeRange = {
- min: '2017-10-05T20:31:48.000Z',
- max: '2017-10-05T20:35:12.000Z',
- };
-
- before('load archive', () => {
- return setup(archive);
- });
+ describe('index detail - metricbeat and package', () => {
+ ['mb', 'package'].forEach((source) => {
+ describe(`index detail ${source}`, () => {
+ const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+ const timeRange = {
+ min: '2017-10-05T20:31:48.000Z',
+ max: '2017-10-05T20:35:12.000Z',
+ };
 
- after('unload archive', () => {
- return tearDown();
- });
+  before('load archive', () => {
+  return setup(archive);
+  });
 
- it('should summarize index with chart metrics data for the non-advanced view', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- is_advanced: false,
- })
- .expect(200);
-
- expect(body).to.eql(indexDetailFixture);
- });
+ after('unload archive', () => {
+ return tearDown(archive);
+ });
+
+ it('should summarize index with chart metrics data for the non-advanced view', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ is_advanced: false,
+ })
+ .expect(200);
+
+ expect(body).to.eql(indexDetailFixture);
+ });
+
+ it('should summarize index with chart metrics data for the advanced view', async () => {
+ const { body } = await supertest
+ .post(
+ '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+ )
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ timeRange,
+ is_advanced: true,
+ })
+ .expect(200);
 
- it('should summarize index with chart metrics data for the advanced view', async () => {
- const { body } = await supertest
- .post(
- '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
- )
- .set('kbn-xsrf', 'xxx')
- .send({
- timeRange,
- is_advanced: true,
- })
- .expect(200);
-
- expect(body).to.eql(indexDetailAdvancedFixture);
+ expect(body).to.eql(indexDetailAdvancedFixture);
+ });
+ });
  });
  });
 }