[8.7] [functional tests] split fleet_api_integration config into smal…

…ler ones (#156407) (#156851) # Backport This will backport the following commits from `main` to `8.7`: - [[functional tests] split fleet_api_integration config into smaller ones (#156407)](#156407)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>"}},{"branch":"8.7","label":"v8.7.2","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"#156407 tests] split fleet_api_integration config into smaller ones (#156407)\n\n## Summary\r\n\r\nRuntime for `fleet_api_integration/config` crossed the 38 min limit.\r\nIdeally config runtime should be <10 min so that pipeline can quickly\r\nretry failed configs (we do 1 retry on CI)\r\n\r\n<img width=\"1600\" alt=\"image\"\r\nsrc=\"https://user-images.githubusercontent.com/10977896/235707744-3120e1e9-4882-493f-9ee0-86016a765401.png\">\r\n\r\n\r\nThis PR splits the existing config into few smaller ones:\r\n\r\n```\r\nx-pack/test/fleet_api_integration/config.agent.ts 15m 15s\r\nx-pack/test/fleet_api_integration/config.agent_policy.ts 4m 10s\r\nx-pack/test/fleet_api_integration/config.epm.ts 8m 12s\r\nx-pack/test/fleet_api_integration/config.package_policy.ts 10m 54s\r\n// combines multiple test files\r\nx-pack/test/fleet_api_integration/config.fleet.ts 5m 2s\r\n```","sha":"1c777a2e702f7c5ae9c1b86063515c0a27368efb"}}]}] BACKPORT-->
elastic · May 5, 2023 · 8140890 · 8140890
1 parent d77b101
commit 8140890
Show file tree

Hide file tree

Showing 13 changed files with 156 additions and 38 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -12,6 +12,7 @@ disabled:
  - test/server_integration/config.base.js
  - x-pack/test/functional_with_es_ssl/config.base.ts
  - x-pack/test/api_integration/config.ts
+ - x-pack/test/fleet_api_integration/config.base.ts
 
  # QA suites that are run out-of-band
  - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -201,7 +202,11 @@ enabled:
  - x-pack/test/encrypted_saved_objects_api_integration/config.ts
  - x-pack/test/endpoint_api_integration_no_ingest/config.ts
  - x-pack/test/examples/config.ts
- - x-pack/test/fleet_api_integration/config.ts
+ - x-pack/test/fleet_api_integration/config.agent.ts
+ - x-pack/test/fleet_api_integration/config.agent_policy.ts
+ - x-pack/test/fleet_api_integration/config.epm.ts
+ - x-pack/test/fleet_api_integration/config.fleet.ts
+ - x-pack/test/fleet_api_integration/config.package_policy.ts
  - x-pack/test/fleet_functional/config.ts
  - x-pack/test/ftr_apis/security_and_spaces/config.ts
  - x-pack/test/functional_basic/apps/ml/permissions/config.ts

diff --git a/x-pack/test/fleet_api_integration/apis/agents/index.js b/x-pack/test/fleet_api_integration/apis/agents/index.js
@@ -5,8 +5,13 @@
  * 2.0.
  */
 
-export default function loadTests({ loadTestFile }) {
- describe('Fleet Endpoints', () => {
+import { setupTestUsers } from '../test_users';
+
+export default function loadTests({ loadTestFile, getService }) {
+ describe('Agents', () => {
+ before(async () => {
+ await setupTestUsers(getService('security'));
+ });
  loadTestFile(require.resolve('./delete'));
  loadTestFile(require.resolve('./list'));
  loadTestFile(require.resolve('./unenroll'));

diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js
@@ -5,8 +5,13 @@
  * 2.0.
  */
 
-export default function loadTests({ loadTestFile }) {
+import { setupTestUsers } from '../test_users';
+
+export default function loadTests({ loadTestFile, getService }) {
  describe('EPM Endpoints', () => {
+ before(async () => {
+ await setupTestUsers(getService('security'));
+ });
  loadTestFile(require.resolve('./delete'));
  loadTestFile(require.resolve('./list'));
  loadTestFile(require.resolve('./setup'));

diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts b/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts
@@ -9,7 +9,7 @@ import expect from '@kbn/expect';
 import fs from 'fs/promises';
 import path from 'path';
 
-import { BUNDLED_PACKAGE_DIR } from '../../config';
+import { BUNDLED_PACKAGE_DIR } from '../../config.base';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';

diff --git a/x-pack/test/fleet_api_integration/apis/index.js b/x-pack/test/fleet_api_integration/apis/index.js
@@ -8,59 +8,43 @@
 import { setupTestUsers } from './test_users';
 
 export default function ({ loadTestFile, getService }) {
+ // total runtime ~ 4m
  describe('Fleet Endpoints', function () {
  before(async () => {
  await setupTestUsers(getService('security'));
  });
 
- // EPM
- loadTestFile(require.resolve('./epm'));
-
  // Fleet setup
- loadTestFile(require.resolve('./fleet_setup'));
-
- // Agents
- loadTestFile(require.resolve('./agents'));
+ loadTestFile(require.resolve('./fleet_setup')); // ~ 6s
 
  // Enrollment API keys
- loadTestFile(require.resolve('./enrollment_api_keys/crud'));
-
- // Package policies
- loadTestFile(require.resolve('./package_policy/create'));
- loadTestFile(require.resolve('./package_policy/update'));
- loadTestFile(require.resolve('./package_policy/get'));
- loadTestFile(require.resolve('./package_policy/delete'));
- loadTestFile(require.resolve('./package_policy/upgrade'));
- loadTestFile(require.resolve('./package_policy/input_package_create_upgrade'));
-
- // Agent policies
- loadTestFile(require.resolve('./agent_policy'));
+ loadTestFile(require.resolve('./enrollment_api_keys/crud')); // ~ 20s
 
  // Data Streams
- loadTestFile(require.resolve('./data_streams'));
+ loadTestFile(require.resolve('./data_streams')); // ~ 20s
 
  // Settings
- loadTestFile(require.resolve('./settings'));
+ loadTestFile(require.resolve('./settings')); // ~ 7s
 
  // Service tokens
- loadTestFile(require.resolve('./service_tokens'));
+ loadTestFile(require.resolve('./service_tokens')); // ~ 2s
 
  // Outputs
- loadTestFile(require.resolve('./outputs'));
+ loadTestFile(require.resolve('./outputs')); // ~ 1m 30s
 
  // Download sources
- loadTestFile(require.resolve('./download_sources'));
+ loadTestFile(require.resolve('./download_sources')); // ~ 15s
 
  // Telemetry
- loadTestFile(require.resolve('./fleet_telemetry'));
+ loadTestFile(require.resolve('./fleet_telemetry')); // ~ 30s
 
  // Integrations
- loadTestFile(require.resolve('./integrations'));
+ loadTestFile(require.resolve('./integrations')); // ~ 8s
 
  // Fleet server hosts
- loadTestFile(require.resolve('./fleet_server_hosts/crud'));
+ loadTestFile(require.resolve('./fleet_server_hosts/crud')); // ~ 9s
 
  // Fleet proxies
- loadTestFile(require.resolve('./fleet_proxies/crud'));
+ loadTestFile(require.resolve('./fleet_proxies/crud')); // ~ 20s
  });
 }
diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/index.js b/x-pack/test/fleet_api_integration/apis/package_policy/index.js
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { setupTestUsers } from '../test_users';
+
+export default function loadTests({ loadTestFile, getService }) {
+ describe('Package policies', () => {
+ before(async () => {
+ await setupTestUsers(getService('security'));
+ });
+ loadTestFile(require.resolve('./create'));
+ loadTestFile(require.resolve('./update'));
+ loadTestFile(require.resolve('./get'));
+
+ loadTestFile(require.resolve('./delete'));
+ loadTestFile(require.resolve('./upgrade'));
+ loadTestFile(require.resolve('./input_package_create_upgrade'));
+ });
+}
diff --git a/x-pack/test/fleet_api_integration/config.agent.ts b/x-pack/test/fleet_api_integration/config.agent.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));
+
+ return {
+ ...baseFleetApiConfig.getAll(),
+ testFiles: [require.resolve('./apis/agents')],
+ junit: {
+ reportName: 'X-Pack Fleet Agent API Integration Tests',
+ },
+ };
+}
diff --git a/x-pack/test/fleet_api_integration/config.agent_policy.ts b/x-pack/test/fleet_api_integration/config.agent_policy.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));
+
+ return {
+ ...baseFleetApiConfig.getAll(),
+ testFiles: [require.resolve('./apis/agent_policy')],
+ junit: {
+ reportName: 'X-Pack Fleet Agent Policy API Integration Tests',
+ },
+ };
+}
diff --git a/x-pack/test/fleet_api_integration/config.ts → ...test/fleet_api_integration/config.base.ts b/x-pack/test/fleet_api_integration/config.ts → ...test/fleet_api_integration/config.base.ts
@@ -41,7 +41,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
  ]);
 
  return {
- testFiles: [require.resolve('./apis')],
  servers: xPackAPITestsConfig.get('servers'),
  dockerServers: defineDockerServersConfig({
  registry: {
@@ -55,9 +54,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
  },
  }),
  services: xPackAPITestsConfig.get('services'),
- junit: {
- reportName: 'X-Pack EPM API Integration Tests',
- },
  esTestCluster: xPackAPITestsConfig.get('esTestCluster'),
  kbnTestServer: {
  ...xPackAPITestsConfig.get('kbnTestServer'),

diff --git a/x-pack/test/fleet_api_integration/config.epm.ts b/x-pack/test/fleet_api_integration/config.epm.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));
+
+ return {
+ ...baseFleetApiConfig.getAll(),
+ testFiles: [require.resolve('./apis/epm')],
+ junit: {
+ reportName: 'X-Pack EPM API Integration Tests',
+ },
+ };
+}
diff --git a/x-pack/test/fleet_api_integration/config.fleet.ts b/x-pack/test/fleet_api_integration/config.fleet.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));
+
+ return {
+ ...baseFleetApiConfig.getAll(),
+ testFiles: [require.resolve('./apis')],
+ junit: {
+ reportName: 'X-Pack Fleet API Integration Tests',
+ },
+ };
+}
diff --git a/x-pack/test/fleet_api_integration/config.package_policy.ts b/x-pack/test/fleet_api_integration/config.package_policy.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));
+
+ return {
+ ...baseFleetApiConfig.getAll(),
+ testFiles: [require.resolve('./apis/package_policy')],
+ junit: {
+ reportName: 'X-Pack Fleet Package Policy API Integration Tests',
+ },
+ };
+}
diff --git a/x-pack/test/security_solution_endpoint_api_int/registry.ts b/x-pack/test/security_solution_endpoint_api_int/registry.ts
@@ -8,7 +8,7 @@
 import path from 'path';
 
 import { defineDockerServersConfig } from '@kbn/test';
-import { dockerImage as ingestDockerImage } from '../fleet_api_integration/config';
+import { dockerImage as ingestDockerImage } from '../fleet_api_integration/config.base';
 
 /**
  * This is used by CI to set the docker registry port