[Fleet] Fixes for various flaky tests (#172218)

## Summary Adds additional retries, unskips a bunch of suites, mocks out some network dependencies for product versions. Will kick off some flaky test runner jobs for this PR. Closes #162545 Ref #138905 Closes #161624 Closes #171160 Closes #171536 Closes #171541 Closes #172114 Closes #171052 Closes #172129 Closes #172128 Closes #172127 --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Dec 5, 2023 · 7655a88 · 7655a88
1 parent 94bb21e
commit 7655a88
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 22 deletions.
diff --git a/x-pack/plugins/fleet/server/integration_tests/ha_setup.test.ts b/x-pack/plugins/fleet/server/integration_tests/ha_setup.test.ts
@@ -141,8 +141,7 @@ describe('Fleet setup preconfiguration with multiple instances Kibana', () => {
     await stopServers();
   });
 
-  // FLAKY: https://github.com/elastic/kibana/issues/142496
-  describe.skip('preconfiguration setup', () => {
+  describe('preconfiguration setup', () => {
     it('sets up Fleet correctly with single Kibana instance', async () => {
       await addRoots(1);
       const [root1Start] = await startRoots();

diff --git a/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts b/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts
@@ -106,7 +106,7 @@ export function useDockerRegistry() {
   beforeAll(async () => {
     jest.setTimeout(BEFORE_SETUP_TIMEOUT);
     await pRetry(() => pullDockerImage(), {
-      retries: 3,
+      retries: 5,
     });
 
     await pRetry(() => startDockerRegistryServer(), {

diff --git a/x-pack/plugins/fleet/server/integration_tests/upgrade_agent_policy_schema_version.test.ts b/x-pack/plugins/fleet/server/integration_tests/upgrade_agent_policy_schema_version.test.ts
@@ -122,8 +122,7 @@ describe('upgrade agent policy schema version', () => {
     await stopServers();
   });
 
-  // FLAKY: https://github.com/elastic/kibana/issues/142347
-  describe.skip('with package installed with outdated schema version', () => {
+  describe('with package installed with outdated schema version', () => {
     let soClient: SavedObjectsClientContract;
     let esClient: ElasticsearchClient;
 

diff --git a/x-pack/plugins/fleet/server/services/agents/crud.test.ts b/x-pack/plugins/fleet/server/services/agents/crud.test.ts
@@ -27,6 +27,14 @@ jest.mock('../audit_logging');
 jest.mock('../../../common/services/is_agent_upgradeable', () => ({
   isAgentUpgradeable: jest.fn().mockImplementation((agent: Agent) => agent.id.includes('up')),
 }));
+jest.mock('./versions', () => {
+  return {
+    getAvailableVersions: jest
+      .fn()
+      .mockResolvedValue(['8.4.0', '8.5.0', '8.6.0', '8.7.0', '8.8.0']),
+    getLatestAvailableVersion: jest.fn().mockResolvedValue('8.8.0'),
+  };
+});
 
 const mockedAuditLoggingService = auditLoggingService as jest.Mocked<typeof auditLoggingService>;
 
@@ -151,8 +159,7 @@ describe('Agents CRUD test', () => {
     });
   });
 
-  // FLAKY: https://github.com/elastic/kibana/issues/171541
-  describe.skip('getAgentsByKuery', () => {
+  describe('getAgentsByKuery', () => {
     it('should return upgradeable on first page', async () => {
       searchMock
         .mockImplementationOnce(() => Promise.resolve(getEsResponse(['1', '2', '3', '4', '5'], 7)))

diff --git a/x-pack/plugins/fleet/server/services/agents/upgrade.test.ts b/x-pack/plugins/fleet/server/services/agents/upgrade.test.ts
@@ -15,6 +15,15 @@ import { sendUpgradeAgentsActions } from './upgrade';
 import { createClientMock } from './action.mock';
 import { getRollingUpgradeOptions, upgradeBatch } from './upgrade_action_runner';
 
+jest.mock('./versions', () => {
+  return {
+    getAvailableVersions: jest
+      .fn()
+      .mockResolvedValue(['8.4.0', '8.5.0', '8.6.0', '8.7.0', '8.8.0']),
+    getLatestAvailableVersion: jest.fn().mockResolvedValue('8.8.0'),
+  };
+});
+
 jest.mock('./action_status', () => {
   return {
     getCancelledActions: jest.fn().mockResolvedValue([
@@ -25,11 +34,7 @@ jest.mock('./action_status', () => {
   };
 });
 
-// FLAKY: https://github.com/elastic/kibana/issues/171052
-// FLAKY: https://github.com/elastic/kibana/issues/172114
-// FLAKY: https://github.com/elastic/kibana/issues/171536
-// FLAKY: https://github.com/elastic/kibana/issues/171160
-describe.skip('sendUpgradeAgentsActions (plural)', () => {
+describe('sendUpgradeAgentsActions (plural)', () => {
   beforeEach(async () => {
     appContextService.start(createAppContextStartContractMock());
   });

diff --git a/x-pack/test/fleet_api_integration/apis/agents/reassign.ts b/x-pack/test/fleet_api_integration/apis/agents/reassign.ts
@@ -200,7 +200,7 @@ export default function (providerContext: FtrProviderContext) {
         await new Promise((resolve, reject) => {
           let attempts = 0;
           const intervalId = setInterval(async () => {
-            if (attempts > 2) {
+            if (attempts > 5) {
               clearInterval(intervalId);
               reject(new Error('action timed out'));
             }

diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_integration_in_multiple_spaces.ts b/x-pack/test/fleet_api_integration/apis/epm/install_integration_in_multiple_spaces.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 import expect from '@kbn/expect';
+import pRetry from 'p-retry';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';
@@ -67,7 +68,6 @@ export default function (providerContext: FtrProviderContext) {
       })
       .catch(() => {});
 
-  // FLAKY: https://github.com/elastic/kibana/issues/161624
   describe.skip('When installing system integration in multiple spaces', async () => {
     skipIfNoDockerRegistry(providerContext);
     setupFleetAndAgents(providerContext);
@@ -89,15 +89,24 @@ export default function (providerContext: FtrProviderContext) {
 
     it('should install kibana assets', async function () {
       // These are installed from Fleet along with every package
-      const resIndexPatternLogs = await kibanaServer.savedObjects.get({
-        type: 'index-pattern',
-        id: 'logs-*',
-      });
+      const resIndexPatternLogs = await pRetry(
+        () =>
+          kibanaServer.savedObjects.get({
+            type: 'index-pattern',
+            id: 'logs-*',
+          }),
+        { retries: 3 }
+      );
       expect(resIndexPatternLogs.id).equal('logs-*');
-      const resIndexPatternMetrics = await kibanaServer.savedObjects.get({
-        type: 'index-pattern',
-        id: 'metrics-*',
-      });
+
+      const resIndexPatternMetrics = await pRetry(
+        () =>
+          kibanaServer.savedObjects.get({
+            type: 'index-pattern',
+            id: 'metrics-*',
+          }),
+        { retries: 3 }
+      );
       expect(resIndexPatternMetrics.id).equal('metrics-*');
     });