Skip to content

Commit

Permalink
[functional tests] split fleet_api_integration config into smaller on…
Browse files Browse the repository at this point in the history
…es (elastic#156407)

## Summary

Runtime for `fleet_api_integration/config` crossed the 38 min limit.
Ideally config runtime should be <10 min so that pipeline can quickly
retry failed configs (we do 1 retry on CI)

<img width="1600" alt="image"
src="https://user-images.githubusercontent.com/10977896/235707744-3120e1e9-4882-493f-9ee0-86016a765401.png">

This PR splits the existing config into few smaller ones:

```
x-pack/test/fleet_api_integration/config.agent.ts 15m 15s
x-pack/test/fleet_api_integration/config.agent_policy.ts 4m 10s
x-pack/test/fleet_api_integration/config.epm.ts 8m 12s
x-pack/test/fleet_api_integration/config.package_policy.ts 10m 54s
// combines multiple test files
x-pack/test/fleet_api_integration/config.fleet.ts 5m 2s
```

(cherry picked from commit 1c777a2)

# Conflicts:
#	x-pack/test/fleet_api_integration/apis/index.js
  • Loading branch information
dmlemeshko committed May 5, 2023
1 parent d77b101 commit 49d7c59
Show file tree
Hide file tree
Showing 13 changed files with 156 additions and 38 deletions.
7 changes: 6 additions & 1 deletion .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ disabled:
- test/server_integration/config.base.js
- x-pack/test/functional_with_es_ssl/config.base.ts
- x-pack/test/api_integration/config.ts
- x-pack/test/fleet_api_integration/config.base.ts

# QA suites that are run out-of-band
- x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
Expand Down Expand Up @@ -201,7 +202,11 @@ enabled:
- x-pack/test/encrypted_saved_objects_api_integration/config.ts
- x-pack/test/endpoint_api_integration_no_ingest/config.ts
- x-pack/test/examples/config.ts
- x-pack/test/fleet_api_integration/config.ts
- x-pack/test/fleet_api_integration/config.agent.ts
- x-pack/test/fleet_api_integration/config.agent_policy.ts
- x-pack/test/fleet_api_integration/config.epm.ts
- x-pack/test/fleet_api_integration/config.fleet.ts
- x-pack/test/fleet_api_integration/config.package_policy.ts
- x-pack/test/fleet_functional/config.ts
- x-pack/test/ftr_apis/security_and_spaces/config.ts
- x-pack/test/functional_basic/apps/ml/permissions/config.ts
Expand Down
9 changes: 7 additions & 2 deletions x-pack/test/fleet_api_integration/apis/agents/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,13 @@
* 2.0.
*/

export default function loadTests({ loadTestFile }) {
describe('Fleet Endpoints', () => {
import { setupTestUsers } from '../test_users';

export default function loadTests({ loadTestFile, getService }) {
describe('Agents', () => {
before(async () => {
await setupTestUsers(getService('security'));
});
loadTestFile(require.resolve('./delete'));
loadTestFile(require.resolve('./list'));
loadTestFile(require.resolve('./unenroll'));
Expand Down
7 changes: 6 additions & 1 deletion x-pack/test/fleet_api_integration/apis/epm/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,13 @@
* 2.0.
*/

export default function loadTests({ loadTestFile }) {
import { setupTestUsers } from '../test_users';

export default function loadTests({ loadTestFile, getService }) {
describe('EPM Endpoints', () => {
before(async () => {
await setupTestUsers(getService('security'));
});
loadTestFile(require.resolve('./delete'));
loadTestFile(require.resolve('./list'));
loadTestFile(require.resolve('./setup'));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import expect from '@kbn/expect';
import fs from 'fs/promises';
import path from 'path';

import { BUNDLED_PACKAGE_DIR } from '../../config';
import { BUNDLED_PACKAGE_DIR } from '../../config.base';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { skipIfNoDockerRegistry } from '../../helpers';
import { setupFleetAndAgents } from '../agents/services';
Expand Down
40 changes: 12 additions & 28 deletions x-pack/test/fleet_api_integration/apis/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,59 +8,43 @@
import { setupTestUsers } from './test_users';

export default function ({ loadTestFile, getService }) {
// total runtime ~ 4m
describe('Fleet Endpoints', function () {
before(async () => {
await setupTestUsers(getService('security'));
});

// EPM
loadTestFile(require.resolve('./epm'));

// Fleet setup
loadTestFile(require.resolve('./fleet_setup'));

// Agents
loadTestFile(require.resolve('./agents'));
loadTestFile(require.resolve('./fleet_setup')); // ~ 6s

// Enrollment API keys
loadTestFile(require.resolve('./enrollment_api_keys/crud'));

// Package policies
loadTestFile(require.resolve('./package_policy/create'));
loadTestFile(require.resolve('./package_policy/update'));
loadTestFile(require.resolve('./package_policy/get'));
loadTestFile(require.resolve('./package_policy/delete'));
loadTestFile(require.resolve('./package_policy/upgrade'));
loadTestFile(require.resolve('./package_policy/input_package_create_upgrade'));

// Agent policies
loadTestFile(require.resolve('./agent_policy'));
loadTestFile(require.resolve('./enrollment_api_keys/crud')); // ~ 20s

// Data Streams
loadTestFile(require.resolve('./data_streams'));
loadTestFile(require.resolve('./data_streams')); // ~ 20s

// Settings
loadTestFile(require.resolve('./settings'));
loadTestFile(require.resolve('./settings')); // ~ 7s

// Service tokens
loadTestFile(require.resolve('./service_tokens'));
loadTestFile(require.resolve('./service_tokens')); // ~ 2s

// Outputs
loadTestFile(require.resolve('./outputs'));
loadTestFile(require.resolve('./outputs')); // ~ 1m 30s

// Download sources
loadTestFile(require.resolve('./download_sources'));
loadTestFile(require.resolve('./download_sources')); // ~ 15s

// Telemetry
loadTestFile(require.resolve('./fleet_telemetry'));
loadTestFile(require.resolve('./fleet_telemetry')); // ~ 30s

// Integrations
loadTestFile(require.resolve('./integrations'));
loadTestFile(require.resolve('./integrations')); // ~ 8s

// Fleet server hosts
loadTestFile(require.resolve('./fleet_server_hosts/crud'));
loadTestFile(require.resolve('./fleet_server_hosts/crud')); // ~ 9s

// Fleet proxies
loadTestFile(require.resolve('./fleet_proxies/crud'));
loadTestFile(require.resolve('./fleet_proxies/crud')); // ~ 20s
});
}
23 changes: 23 additions & 0 deletions x-pack/test/fleet_api_integration/apis/package_policy/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { setupTestUsers } from '../test_users';

export default function loadTests({ loadTestFile, getService }) {
describe('Package policies', () => {
before(async () => {
await setupTestUsers(getService('security'));
});
loadTestFile(require.resolve('./create'));
loadTestFile(require.resolve('./update'));
loadTestFile(require.resolve('./get'));

loadTestFile(require.resolve('./delete'));
loadTestFile(require.resolve('./upgrade'));
loadTestFile(require.resolve('./input_package_create_upgrade'));
});
}
20 changes: 20 additions & 0 deletions x-pack/test/fleet_api_integration/config.agent.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));

return {
...baseFleetApiConfig.getAll(),
testFiles: [require.resolve('./apis/agents')],
junit: {
reportName: 'X-Pack Fleet Agent API Integration Tests',
},
};
}
20 changes: 20 additions & 0 deletions x-pack/test/fleet_api_integration/config.agent_policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));

return {
...baseFleetApiConfig.getAll(),
testFiles: [require.resolve('./apis/agent_policy')],
junit: {
reportName: 'X-Pack Fleet Agent Policy API Integration Tests',
},
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
]);

return {
testFiles: [require.resolve('./apis')],
servers: xPackAPITestsConfig.get('servers'),
dockerServers: defineDockerServersConfig({
registry: {
Expand All @@ -55,9 +54,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
},
}),
services: xPackAPITestsConfig.get('services'),
junit: {
reportName: 'X-Pack EPM API Integration Tests',
},
esTestCluster: xPackAPITestsConfig.get('esTestCluster'),
kbnTestServer: {
...xPackAPITestsConfig.get('kbnTestServer'),
Expand Down
20 changes: 20 additions & 0 deletions x-pack/test/fleet_api_integration/config.epm.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));

return {
...baseFleetApiConfig.getAll(),
testFiles: [require.resolve('./apis/epm')],
junit: {
reportName: 'X-Pack EPM API Integration Tests',
},
};
}
20 changes: 20 additions & 0 deletions x-pack/test/fleet_api_integration/config.fleet.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));

return {
...baseFleetApiConfig.getAll(),
testFiles: [require.resolve('./apis')],
junit: {
reportName: 'X-Pack Fleet API Integration Tests',
},
};
}
20 changes: 20 additions & 0 deletions x-pack/test/fleet_api_integration/config.package_policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts'));

return {
...baseFleetApiConfig.getAll(),
testFiles: [require.resolve('./apis/package_policy')],
junit: {
reportName: 'X-Pack Fleet Package Policy API Integration Tests',
},
};
}
2 changes: 1 addition & 1 deletion x-pack/test/security_solution_endpoint_api_int/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import path from 'path';

import { defineDockerServersConfig } from '@kbn/test';
import { dockerImage as ingestDockerImage } from '../fleet_api_integration/config';
import { dockerImage as ingestDockerImage } from '../fleet_api_integration/config.base';

/**
* This is used by CI to set the docker registry port
Expand Down

0 comments on commit 49d7c59

Please sign in to comment.