Revert "[ftr] remove testMetadata and maintain a unique lifecycle ins…

…tance per run"

This reverts commit d2b4fdb.

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -98,7 +98,11 @@ export function runFtrCli() {
           });
         }
 
-        process.exit();
+        try {
+          await functionalTestRunner.close();
+        } finally {
+          process.exit();
+        }
       };
 
       process.on('unhandledRejection', (err) =>

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -14,6 +14,8 @@ import { REPO_ROOT } from '@kbn/utils';
 import { Suite, Test } from './fake_mocha_types';
 import {
   Lifecycle,
+  LifecyclePhase,
+  TestMetadata,
   readConfigFile,
   ProviderCollection,
   Providers,
@@ -28,13 +30,23 @@ import {
 import { createEsClientForFtrConfig } from '../es';
 
 export class FunctionalTestRunner {
+  public readonly lifecycle = new Lifecycle();
+  public readonly testMetadata = new TestMetadata(this.lifecycle);
+  private closed = false;
+
   private readonly esVersion: EsVersion;
   constructor(
     private readonly log: ToolingLog,
     private readonly configFile: string,
     private readonly configOverrides: any,
     esVersion?: string | EsVersion
   ) {
+    for (const [key, value] of Object.entries(this.lifecycle)) {
+      if (value instanceof LifecyclePhase) {
+        value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
+        value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
+      }
+    }
     this.esVersion =
       esVersion === undefined
         ? EsVersion.getDefault()
@@ -46,8 +58,8 @@ export class FunctionalTestRunner {
   async run() {
     const testStats = await this.getTestStats();
 
-    return await this.runHarness(async (config, lifecycle, coreProviders) => {
-      SuiteTracker.startTracking(lifecycle, this.configFile);
+    return await this._run(async (config, coreProviders) => {
+      SuiteTracker.startTracking(this.lifecycle, this.configFile);
 
       const realServices =
         !!config.get('testRunner') ||
@@ -89,7 +101,7 @@ export class FunctionalTestRunner {
       }
 
       const mocha = await setupMocha(
-        lifecycle,
+        this.lifecycle,
         this.log,
         config,
         providers,
@@ -107,10 +119,10 @@ export class FunctionalTestRunner {
         return this.simulateMochaDryRun(mocha);
       }
 
-      await lifecycle.beforeTests.trigger(mocha.suite);
+      await this.lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 
-      return await runTests(lifecycle, mocha);
+      return await runTests(this.lifecycle, mocha);
     });
   }
 
@@ -142,13 +154,13 @@ export class FunctionalTestRunner {
   }
 
   async getTestStats() {
-    return await this.runHarness(async (config, lifecycle, coreProviders) => {
+    return await this._run(async (config, coreProviders) => {
       if (config.get('testRunner')) {
         throw new Error('Unable to get test stats for config that uses a custom test runner');
       }
 
       const providers = this.getStubProviderCollection(config, coreProviders);
-      const mocha = await setupMocha(lifecycle, this.log, config, providers, this.esVersion);
+      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
 
       const queue = new Set([mocha.suite]);
       const allTests: Test[] = [];
@@ -204,11 +216,10 @@ export class FunctionalTestRunner {
     ]);
   }
 
-  private async runHarness<T = any>(
-    handler: (config: Config, lifecycle: Lifecycle, coreProviders: Providers) => Promise<T>
+  async _run<T = any>(
+    handler: (config: Config, coreProviders: Providers) => Promise<T>
   ): Promise<T> {
     let runErrorOccurred = false;
-    const lifecycle = new Lifecycle(this.log);
 
     try {
       const config = await readConfigFile(
@@ -229,25 +240,26 @@ export class FunctionalTestRunner {
       const dockerServers = new DockerServersService(
         config.get('dockerServers'),
         this.log,
-        lifecycle
+        this.lifecycle
       );
 
       // base level services that functional_test_runner exposes
       const coreProviders = readProviderSpec('Service', {
-        lifecycle: () => lifecycle,
+        lifecycle: () => this.lifecycle,
         log: () => this.log,
+        testMetadata: () => this.testMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
         esVersion: () => this.esVersion,
       });
 
-      return await handler(config, lifecycle, coreProviders);
+      return await handler(config, coreProviders);
     } catch (runError) {
       runErrorOccurred = true;
       throw runError;
     } finally {
       try {
-        await lifecycle.cleanup.trigger();
+        await this.close();
       } catch (closeError) {
         if (runErrorOccurred) {
           this.log.error('failed to close functional_test_runner');
@@ -260,6 +272,13 @@ export class FunctionalTestRunner {
     }
   }
 
+  async close() {
+    if (this.closed) return;
+
+    this.closed = true;
+    await this.lifecycle.cleanup.trigger();
+  }
+
   simulateMochaDryRun(mocha: any) {
     interface TestEntry {
       file: string;

packages/kbn-test/src/functional_test_runner/index.ts

@@ -15,6 +15,7 @@ export {
   Lifecycle,
   LifecyclePhase,
 } from './lib';
+export type { ScreenshotRecord } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -12,6 +12,7 @@ export { readConfigFile, Config } from './config';
 export * from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';
+export * from './test_metadata';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -6,48 +6,29 @@
  * Side Public License, v 1.
  */
 
-import * as Rx from 'rxjs';
-import { ToolingLog } from '@kbn/tooling-log';
-
 import { LifecyclePhase } from './lifecycle_phase';
 
 import { Suite, Test } from '../fake_mocha_types';
 
 export class Lifecycle {
-  /** root subscription to cleanup lifecycle phases when lifecycle completes */
-  private readonly sub = new Rx.Subscription();
-
   /** lifecycle phase that will run handlers once before tests execute */
-  public readonly beforeTests = new LifecyclePhase<[Suite]>(this.sub, {
+  public readonly beforeTests = new LifecyclePhase<[Suite]>({
     singular: true,
   });
   /** lifecycle phase that runs handlers before each runnable (test and hooks) */
-  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>(this.sub);
+  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
   /** lifecycle phase that runs handlers before each suite */
-  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>(this.sub);
+  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
   /** lifecycle phase that runs handlers before each test */
-  public readonly beforeEachTest = new LifecyclePhase<[Test]>(this.sub);
+  public readonly beforeEachTest = new LifecyclePhase<[Test]>();
   /** lifecycle phase that runs handlers after each suite */
-  public readonly afterTestSuite = new LifecyclePhase<[Suite]>(this.sub);
+  public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
   /** lifecycle phase that runs handlers after a test fails */
-  public readonly testFailure = new LifecyclePhase<[Error, Test]>(this.sub);
+  public readonly testFailure = new LifecyclePhase<[Error, Test]>();
   /** lifecycle phase that runs handlers after a hook fails */
-  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>(this.sub);
+  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
   /** lifecycle phase that runs handlers at the very end of execution */
-  public readonly cleanup = new LifecyclePhase<[]>(this.sub, {
+  public readonly cleanup = new LifecyclePhase<[]>({
     singular: true,
   });
-
-  constructor(log: ToolingLog) {
-    for (const [name, phase] of Object.entries(this)) {
-      if (phase instanceof LifecyclePhase) {
-        phase.before$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
-        phase.after$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
-      }
-    }
-
-    this.cleanup.after$.subscribe(() => {
-      this.sub.unsubscribe();
-    });
-  }
 }

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.test.ts

@@ -26,7 +26,7 @@ describe('with randomness', () => {
   });
 
   it('calls handlers in random order', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription());
+    const phase = new LifecyclePhase();
     const order: string[] = [];
 
     phase.add(
@@ -69,7 +69,7 @@ describe('without randomness', () => {
   afterEach(() => jest.restoreAllMocks());
 
   it('calls all handlers and throws first error', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription());
+    const phase = new LifecyclePhase();
     const fn1 = jest.fn();
     phase.add(fn1);
 
@@ -88,7 +88,7 @@ describe('without randomness', () => {
   });
 
   it('triggers before$ just before calling handler and after$ once it resolves', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription());
+    const phase = new LifecyclePhase();
     const order: string[] = [];
 
     const beforeSub = jest.fn(() => order.push('before'));
@@ -116,7 +116,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ and after$ if phase is singular', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
+    const phase = new LifecyclePhase({ singular: true });
 
     const beforeNotifs: Array<Rx.Notification<unknown>> = [];
     phase.before$.pipe(materialize()).subscribe((n) => beforeNotifs.push(n));
@@ -160,7 +160,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
+    const phase = new LifecyclePhase({ singular: true });
     await phase.trigger();
 
     await expect(phase.before$.pipe(materialize(), toArray()).toPromise()).resolves
@@ -177,7 +177,7 @@ describe('without randomness', () => {
   });
 
   it('replays after$ event subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
+    const phase = new LifecyclePhase({ singular: true });
     await phase.trigger();
 
     await expect(phase.after$.pipe(materialize(), toArray()).toPromise()).resolves

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.ts

@@ -26,7 +26,6 @@ export class LifecyclePhase<Args extends readonly any[]> {
   public readonly after$: Rx.Observable<void>;
 
   constructor(
-    sub: Rx.Subscription,
     private readonly options: {
       singular?: boolean;
     } = {}
@@ -36,12 +35,6 @@ export class LifecyclePhase<Args extends readonly any[]> {
 
     this.afterSubj = this.options.singular ? new Rx.ReplaySubject<void>(1) : new Rx.Subject<void>();
     this.after$ = this.afterSubj.asObservable();
-
-    sub.add(() => {
-      this.beforeSubj.complete();
-      this.afterSubj.complete();
-      this.handlers.length = 0;
-    });
   }
 
   public add(fn: (...args: Args) => Promise<void> | void) {

packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js

@@ -69,9 +69,6 @@ function setup({ include, exclude, esVersion }) {
         info(...args) {
           history.push(`info: ${format(...args)}`);
         },
-        debug(...args) {
-          history.push(`debg: ${format(...args)}`);
-        },
       },
       mocha,
       include,
@@ -224,7 +221,7 @@ it(`excludes tests which don't meet the esVersionRequirement`, async () => {
 
   expect(history).toMatchInlineSnapshot(`
     Array [
-      "debg: Only running suites which are compatible with ES version 9.0.0",
+      "info: Only running suites which are compatible with ES version 9.0.0",
       "suite: ",
       "suite: level 1",
       "suite: level 1 level 1a",

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/ci_stats_ftr_reporter.ts

@@ -17,6 +17,7 @@ import {
 
 import { Config } from '../../config';
 import { Runner } from '../../../fake_mocha_types';
+import { TestMetadata, ScreenshotRecord } from '../../test_metadata';
 import { Lifecycle } from '../../lifecycle';
 import { getSnapshotOfRunnableLogs } from '../../../../mocha';
 
@@ -35,6 +36,7 @@ interface Runnable {
   file: string;
   title: string;
   parent: Suite;
+  _screenshots?: ScreenshotRecord[];
 }
 
 function getHookType(hook: Runnable): CiStatsTestType {
@@ -58,11 +60,13 @@ export function setupCiStatsFtrTestGroupReporter({
   config,
   lifecycle,
   runner,
+  testMetadata,
   reporter,
 }: {
   config: Config;
   lifecycle: Lifecycle;
   runner: Runner;
+  testMetadata: TestMetadata;
   reporter: CiStatsReporter;
 }) {
   const testGroupType = process.env.TEST_GROUP_TYPE_FUNCTIONAL;
@@ -107,6 +111,10 @@ export function setupCiStatsFtrTestGroupReporter({
       type,
       error: error?.stack,
       stdout: getSnapshotOfRunnableLogs(runnable),
+      screenshots: testMetadata.getScreenshots(runnable).map((s) => ({
+        base64Png: s.base64Png,
+        name: s.name,
+      })),
     });
   }
 

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -24,6 +24,7 @@ export function MochaReporterProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
   const lifecycle = getService('lifecycle');
+  const testMetadata = getService('testMetadata');
   let originalLogWriters;
   let reporterCaptureStartTime;
 
@@ -60,6 +61,7 @@ export function MochaReporterProvider({ getService }) {
             config,
             lifecycle,
             runner,
+            testMetadata,
           });
         }
       }