Long run monitoring (#904)

* enhance reporters

* adding long run monitoring

* fix eslint error

README.md

@@ -175,11 +175,13 @@ Content
         - [shell](#shell)
     - [Debug Config](#debug-config)
     - [Debug Config v2](#debug-config-v2)
+    - [monitorLongRun](#monitorlongrun)
   - [Commands](#commands)
   - [Menu](#menu)
   - [Troubleshooting](#troubleshooting)
     - [Jest failed to run](#jest-failed-to-run)
     - [I don't see "Jest" in the bottom status bar](#i-dont-see-jest-in-the-bottom-status-bar)
+    - [What to do with "Long Running Tests Warning"](#what-to-do-with-long-running-tests-warning)
     - [The extension seems to consume high CPU](#the-extension-seems-to-consume-high-cpu)
     - [The tests and status do not match or some tests showing question marks unexpectedly?](#the-tests-and-status-do-not-match-or-some-tests-showing-question-marks-unexpectedly)
   - [Want to Contribute?](#want-to-contribute)
@@ -368,9 +370,10 @@ Users can use the following settings to tailor the extension for their environme
 |nodeEnv|Add additional env variables to spawned jest process|null|`"jest.nodeEnv": {"PORT": "9800", "BAR":"true"}` |
 |shell|Custom shell (path or LoginShell) for executing jest|null|`"jest.shell": "/bin/bash"` or `"jest.shell": "powershell"` or `"jest.shell": {"path": "/bin/bash"; args: ["--login"]}`  |
 |[autoRun](#autorun)|Controls when and what tests should be run|undefined|`"jest.autoRun": "off"` or `"jest.autoRun": {"watch": true, "onStartup": ["all-tests"]}` or `"jest.autoRun": false, onSave:"test-only"}`|
+|[rootPath](#rootPath)|The path to your frontend src folder|""|`"jest.rootPath":"packages/app"` or `"jest.rootPath":"/apps/my-app"`|
+|[monitorLongRun](#monitorlongrun)| monitor long running tests based on given threshold in ms|60000|`"jest.monitorLongRun": 120000`|
 |pathToJest :x:|The path to the Jest binary, or an npm/yarn command to run tests|undefined|Please use `jestCommandLine` instead|
 |pathToConfig :x:|The path to your Jest configuration file"|""|Please use `jestCommandLine` instead|
-|[rootPath](#rootPath)|The path to your frontend src folder|""|`"jest.rootPath":"packages/app"` or `"jest.rootPath":"/apps/my-app"`|
 |runAllTestsFirst :x:| Run all tests before starting Jest in watch mode|true|Please use `autoRun` instead|
 |**Editor**|
 |<strike>enableInlineErrorMessages</strike> :x:| Whether errors should be reported inline on a file|--|This is now deprecated in favor of `jest.testExplorer` |
@@ -601,7 +604,15 @@ Currently supported variables:
   ``` 
 
 </details>
+### monitorLongRun
+```ts
+monitorLongRun = number | 'off'
+```
 
+- specify a number (milliseconds) means any run exceeds this threshold will trigger a warning. The number has to be > 0. 
+- specify "off" to disable long-run process monitoring
+
+Default is `"jest.monitorLongRun":60000` (1 minute)
 ## Commands
 
 This extension contributes the following commands and can be accessed via [Command Palette](https://code.visualstudio.com/docs/getstarted/userinterface#_command-palette):
@@ -673,6 +684,16 @@ There could be other causes, such as jest test root path is different from the p
   - your source and tests are not in the project root directory: try [jest.rootPath](#rootPath) to point to that directory instead.
 
   Users can also try to manually activate the extension via command palette: `"Jest: Start All Runners"`
+### What to do with "Long Running Tests Warning"
+The extension monitor excessive test run with ["jest.monitorLongRun"](#monitorlongrun) setting. By default if any runs exceed 60 seconds, a warning message will be shown. 
+- If running the tests with the extension seems to be longer than running it from a terminal, chances are you can use ["jest.autoRun"](#autorun) to optimize it, for example:
+  - for process type "all-tests", you can turn off the all-tests from autoRun.
+  - for process type "watch-tests" or "watch-all-tests", you can maybe turn off watch mode and use "onSave" instead. 
+
+- If the tests are slow even from the terminal, i.e. without the extension, you will need to optimize your tests, feel free to check out [jest troubleshooting](https://jestjs.io/docs/troubleshooting) or other online articles.
+- If the run appeared to hang, i.e. the TestExplorer or statusBar showed test running when it is not. It might be related to this [jest issue](https://github.com/facebook/jest/issues/13187), which should be fixed after release `29.0.2`. If you believe your issue is different, please [file a new issue](https://github.com/jest-community/vscode-jest/issues) so we can take a look.
+
+You can also turn off the monitor or change the threshold with ["jest.monitorLongRun"](#monitorlongrun) to meet your needs. 
 ### The extension seems to consume high CPU 
   By default the extension will run all tests when it is launched followed by a jest watch process. If you have many resource intensive tests or source files that can trigger many tests when changed, this could be the reason. Check out [jest.autoRun](#autorun) to see how you can change and control when and what tests should be run.
 

package.json

@@ -228,6 +228,15 @@
             "enabled": true
           },
           "scope": "resource"
+        },
+        "jest.monitorLongRun": {
+          "markdownDescription": "Enable monitoring for long running test process. See valid [monitorLongRun](https://github.com/jest-community/vscode-jest/blob/master/README.md#monitorLongRun) for details",
+          "type": [
+            "string",
+            "integer"
+          ],
+          "default": 60000,
+          "scope": "resource"
         }
       }
     },

src/JestExt/core.ts

@@ -39,7 +39,7 @@ interface RunTestPickItem extends vscode.QuickPickItem {
   id: DebugTestIdentifier;
 }
 
-type MessageActionType = 'help' | 'wizard' | 'disable-folder';
+type MessageActionType = 'help' | 'wizard' | 'disable-folder' | 'help-long-run';
 
 /** extract lines starts and end with [] */
 export class JestExt {
@@ -156,6 +156,16 @@ export class JestExt {
       },
     };
   }
+  private longRunMessage(event: Extract<JestRunEvent, { type: 'long-run' }>): string {
+    const messages = [`Long Running Tests Warning: Jest process "${event.process.request.type}"`];
+    if (event.numTotalTestSuites != null) {
+      messages.push(`for ${event.numTotalTestSuites} suites`);
+    }
+    messages.push(`has exceeded ${event.threshold}ms.`);
+
+    return messages.join(' ');
+  }
+
   private setupRunEvents(events: JestSessionEvents): void {
     events.onRunEvent.event((event: JestRunEvent) => {
       switch (event.type) {
@@ -194,6 +204,12 @@ export class JestExt {
             this.updateStatusBar({ state: 'done' });
           }
           break;
+        case 'long-run': {
+          const msg = prefixWorkspace(this.extContext, this.longRunMessage(event));
+          messaging.systemWarningMessage(msg, ...this.buildMessageActions(['help-long-run']));
+          this.logging('warn', msg);
+          break;
+        }
       }
     });
   }
@@ -213,6 +229,9 @@ export class JestExt {
             actions.push(this.setupIgnoreAction());
           }
           break;
+        case 'help-long-run':
+          actions.push(messaging.showLongRunTroubleshootingAction);
+          break;
       }
     }
     return actions;

src/JestExt/helper.ts

@@ -10,6 +10,7 @@ import {
   JestExtAutoRunConfig,
   TestExplorerConfig,
   NodeEnv,
+  MonitorLongRun,
 } from '../Settings';
 import { AutoRunMode } from '../StatusBar';
 import { pathToJest, pathToConfig, toFilePath } from '../helpers';
@@ -114,16 +115,12 @@ const isLoginShell = (arg: any): arg is LoginShell =>
   arg && typeof arg.path === 'string' && Array.isArray(arg.args);
 
 const getShell = (config: vscode.WorkspaceConfiguration): string | LoginShell | undefined => {
-  console.log('inside getShell');
-
   const shell = config.get<string | LoginShell>('shell');
 
-  console.log('shell=', shell);
   if (!shell || typeof shell === 'string') {
     return shell;
   }
 
-  console.log('before calling isLoginShell shell=', shell);
   if (isLoginShell(shell)) {
     if (platform() === 'win32') {
       console.error(`LoginShell is not supported for windows currently.`);
@@ -164,6 +161,7 @@ export const getExtensionResourceSettings = (uri: vscode.Uri): PluginResourceSet
     testExplorer: config.get<TestExplorerConfig>('testExplorer') ?? { enabled: true },
     nodeEnv: config.get<NodeEnv | null>('nodeEnv') ?? undefined,
     shell: getShell(config) ?? undefined,
+    monitorLongRun: config.get<MonitorLongRun>('monitorLongRun') ?? undefined,
   };
 };
 

src/JestExt/process-listeners.ts

@@ -6,6 +6,7 @@ import { ListenerSession, ListTestFilesCallback } from './process-session';
 import { isWatchRequest } from './helper';
 import { Logging } from '../logging';
 import { JestRunEvent } from './types';
+import { MonitorLongRun } from '../Settings';
 
 export class AbstractProcessListener {
   protected session: ListenerSession;
@@ -157,8 +158,84 @@ const IS_OUTSIDE_REPOSITORY_REGEXP =
   /Test suite failed to run[\s\S]*fatal:[\s\S]*is outside repository/im;
 const WATCH_IS_NOT_SUPPORTED_REGEXP =
   /^s*--watch is not supported without git\/hg, please use --watchAlls*/im;
+const RUN_EXEC_ERROR = /onRunComplete: execError: (.*)/im;
+const RUN_START_TEST_SUITES_REGEX = /onRunStart: numTotalTestSuites: ((\d)+)/im;
 
+/**
+ * monitor for long test run, default is 1 minute
+ */
+export const DEFAULT_LONG_RUN_THRESHOLD = 60000;
+export class LongRunMonitor {
+  private timer: NodeJS.Timer | undefined;
+  public readonly thresholdMs: number;
+  constructor(private callback: () => void, private logging: Logging, option?: MonitorLongRun) {
+    if (option == null) {
+      this.thresholdMs = DEFAULT_LONG_RUN_THRESHOLD;
+    } else if (typeof option === 'number' && option > 0) {
+      this.thresholdMs = option;
+    } else {
+      this.thresholdMs = -1;
+    }
+    this.timer = undefined;
+  }
+  start(): void {
+    if (this.thresholdMs <= 0) {
+      return;
+    }
+    if (this.timer) {
+      this.logging('warn', `LongRunMonitor is already runninng`);
+      this.cancel();
+    }
+    this.timer = setTimeout(() => {
+      this.callback();
+      this.timer = undefined;
+    }, this.thresholdMs);
+  }
+
+  cancel(): void {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = undefined;
+    }
+  }
+}
+interface RunInfo {
+  process: JestProcess;
+  numTotalTestSuites?: number;
+}
 export class RunTestListener extends AbstractProcessListener {
+  // fire long-run warning once per run
+  private longRunMonitor: LongRunMonitor;
+  private runInfo: RunInfo | undefined;
+
+  constructor(session: ListenerSession) {
+    super(session);
+    this.longRunMonitor = new LongRunMonitor(
+      this.onLongRun.bind(this),
+      this.logging,
+      session.context.settings.monitorLongRun
+    );
+    this.runInfo = undefined;
+  }
+
+  private onLongRun(): void {
+    if (this.runInfo) {
+      this.onRunEvent.fire({
+        type: 'long-run',
+        threshold: this.longRunMonitor.thresholdMs,
+        ...this.runInfo,
+      });
+    }
+  }
+  private runEnded(): void {
+    this.longRunMonitor.cancel();
+    this.runInfo = undefined;
+  }
+  private runStarted(info: RunInfo): void {
+    this.runInfo = info;
+    this.longRunMonitor.start();
+  }
+
   protected get name(): string {
     return 'RunTestListener';
   }
@@ -257,13 +334,37 @@ export class RunTestListener extends AbstractProcessListener {
     this.handleWatchNotSupportedError(process, message);
   }
 
+  private getNumTotalTestSuites(text: string): number | undefined {
+    const matched = text.match(RUN_START_TEST_SUITES_REGEX);
+    if (matched) {
+      const n = Number(matched[1]);
+      if (Number.isInteger(n)) {
+        return n;
+      }
+    }
+  }
   protected onExecutableOutput(process: JestProcess, output: string, raw: string): void {
     if (output.includes('onRunStart')) {
+      this.runStarted({ process, numTotalTestSuites: this.getNumTotalTestSuites(output) });
+
       if (isWatchRequest(process.request)) {
         this.onRunEvent.fire({ type: 'start', process });
       }
     }
     if (output.includes('onRunComplete')) {
+      this.runEnded();
+
+      // possible no output will be generated
+      const matched = output.match(RUN_EXEC_ERROR);
+      if (matched) {
+        this.onRunEvent.fire({
+          type: 'data',
+          process,
+          text: matched[1],
+          newLine: true,
+          isError: true,
+        });
+      }
       if (isWatchRequest(process.request)) {
         this.onRunEvent.fire({ type: 'end', process });
       }
@@ -277,9 +378,11 @@ export class RunTestListener extends AbstractProcessListener {
   protected onTerminalError(process: JestProcess, data: string, raw: string): void {
     this.onRunEvent.fire({ type: 'data', process, text: data, raw, newLine: true, isError: true });
   }
-  protected onProcessExit(_process: JestProcess): void {
-    //override parent method so we will fire run event only when process closed
+  protected onProcessExit(process: JestProcess, code?: number, signal?: string): void {
+    this.runEnded();
+    super.onProcessExit(process, code, signal);
   }
+
   protected onProcessClose(process: JestProcess): void {
     super.onProcessClose(process);
     const error = this.handleWatchProcessCrash(process);

src/JestExt/types.ts

@@ -51,6 +51,7 @@ export type JestRunEvent = RunEventBase &
     | { type: 'start' }
     | { type: 'end' }
     | { type: 'exit'; error?: string }
+    | { type: 'long-run'; threshold: number; numTotalTestSuites?: number }
   );
 export interface JestSessionEvents {
   onRunEvent: vscode.EventEmitter<JestRunEvent>;

src/Settings/index.ts

@@ -27,6 +27,7 @@ export type TestExplorerConfig =
   | { enabled: false }
   | { enabled: true; showClassicStatus?: boolean; showInlineError?: boolean };
 export type NodeEnv = ProjectWorkspace['nodeEnv'];
+export type MonitorLongRun = 'off' | number;
 export interface PluginResourceSettings {
   showTerminalOnLaunch?: boolean;
   autoEnable?: boolean;
@@ -45,6 +46,7 @@ export interface PluginResourceSettings {
   testExplorer: TestExplorerConfig;
   nodeEnv?: NodeEnv;
   shell?: string | LoginShell;
+  monitorLongRun?: MonitorLongRun;
 }
 
 export interface PluginWindowSettings {

src/messaging.ts

@@ -6,6 +6,8 @@ import * as vscode from 'vscode';
 
 export const TROUBLESHOOTING_URL =
   'https://github.com/jest-community/vscode-jest/blob/master/README.md#troubleshooting';
+export const LONG_RUN_TROUBLESHOOTING_URL =
+  'https://github.com/jest-community/vscode-jest/blob/master/README.md#what-to-do-with-long-running-tests-warning';
 
 //
 // internal methods
@@ -58,3 +60,8 @@ export const showTroubleshootingAction: MessageAction = {
   action: () =>
     vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(TROUBLESHOOTING_URL)),
 };
+export const showLongRunTroubleshootingAction: MessageAction = {
+  title: 'Help',
+  action: () =>
+    vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(LONG_RUN_TROUBLESHOOTING_URL)),
+};

src/reporter.ts

@@ -1,13 +1,23 @@
 // Custom Jest reporter used by jest-vscode extension
-class VSCodeJestReporter {
-  onRunStart() {
-    // tslint:disable-next-line: no-console
-    console.log('onRunStart');
+
+import type { AggregatedResult } from '@jest/test-result';
+import { Reporter, Context } from '@jest/reporters';
+
+export default class VSCodeJestReporter implements Reporter {
+  onRunStart(aggregatedResults: AggregatedResult): void {
+    console.log(`onRunStart: numTotalTestSuites: ${aggregatedResults.numTotalTestSuites}`);
   }
-  onRunComplete() {
-    // tslint:disable-next-line: no-console
-    console.log('onRunComplete');
+
+  onRunComplete(_contexts: Set<Context>, results: AggregatedResult): void {
+    // report exec errors that could have prevented Result file being generated
+    // TODO: replace with checking results.runExecError after Jest release https://github.com/facebook/jest/pull/13203
+    if (results.numTotalTests === 0 && results.numTotalTestSuites > 0) {
+      console.log('onRunComplete: execError: no test is run');
+    } else {
+      console.log('onRunComplete');
+    }
+  }
+  getLastError(): Error | undefined {
+    return;
   }
 }
-
-module.exports = VSCodeJestReporter;