feat: List failed tests and print their logs

packages/cli/src/cmds/describeChange.ts

@@ -275,11 +275,15 @@ export const handler = async (argv: any) => {
     if (diagramAsText.diagram.trim().length > 0)
       await writeFile(join(operationDir, [name, 'sequence.txt'].join('.')), diagramAsText.diagram);
 
-    await renderSequenceDiagramPNG(
-      join(operationDir, [name, 'sequence.png'].join('.')),
-      join(operationDir, [name, 'sequence.json'].join('.')),
-      browser
-    );
+    try {
+      await renderSequenceDiagramPNG(
+        join(operationDir, [name, 'sequence.png'].join('.')),
+        join(operationDir, [name, 'sequence.json'].join('.')),
+        browser
+      );
+    } catch (e) {
+      console.warn(`Failed to render sequence diagram for ${operationDir}: ${e}`);
+    }
   }
 
   await Promise.all(
@@ -315,6 +319,21 @@ export const handler = async (argv: any) => {
     })
   );
 
+  if (changeReport.failedTests) {
+    console.log(`${changeReport.failedTests.length} tests failed`);
+    for (const test of changeReport.failedTests) {
+      console.log(`${test.testLocation} (${test.name})`);
+      console.log(`${test.appmapFile}`);
+      console.log(`Log messages:`);
+      for (const logEntry of test.logEntries) {
+        process.stdout.write(`${logEntry.message}`);
+        if (verbose()) console.log(`  ${logEntry.stack.join('\n  ')}`);
+      }
+    }
+  } else {
+    console.log(`All tests passed`);
+  }
+
   /*
 
   const headAppMapFileNameArray = [...headAppMapFileNames].sort();

packages/cli/src/describeChange/AppMapReference.ts

@@ -1,27 +1,27 @@
 import { copyFile, readFile, writeFile } from 'fs/promises';
-import { basename, isAbsolute, join } from 'path';
+import { basename, dirname, isAbsolute, join } from 'path';
 import { existsSync } from 'fs';
 import {
-  Action,
   buildDiagram,
   Diagram,
   format,
   FormatType,
-  isServerRPC,
   SequenceDiagramOptions,
-  ServerRPC,
   Specification,
 } from '@appland/sequence-diagram';
 import { AppMap, CodeObject, buildAppMap, Event } from '@appland/models';
 import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile';
 import { executeCommand } from './executeCommand';
 import { RevisionName } from './RevisionName';
 import { OperationReference } from './OperationReference';
+import { promisify } from 'util';
+import { glob } from 'glob';
 
 type Metadata = {
   sourceLocation: string | undefined;
   appmapName: string | undefined;
   sourcePaths: string[];
+  testStatus?: string;
 };
 
 export class AppMapReference {
@@ -79,6 +79,26 @@ export class AppMapReference {
     return join(...tokens);
   }
 
+  /**
+   * Gets the AppMap file names of failed test cases.
+   */
+  static async listFailedTests(outputDir: string, revisionName: RevisionName): Promise<string[]> {
+    const metadataFiles = await promisify(glob)(`${outputDir}/${revisionName}/*.metadata.json`);
+    const result = new Array<string>();
+    for (const metadataFile of metadataFiles) {
+      const metadata = JSON.parse(await readFile(metadataFile, 'utf-8')) as Metadata;
+      if (metadata.testStatus === 'failed') {
+        result.push(
+          join(
+            dirname(metadataFile),
+            [basename(metadataFile, '.metadata.json'), 'appmap.json'].join('.')
+          )
+        );
+      }
+    }
+    return result;
+  }
+
   async loadSequenceDiagramJSON(revisionName: RevisionName): Promise<Diagram> {
     return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true));
   }
@@ -116,7 +136,7 @@ export class AppMapReference {
     );
   }
 
-  public async restoreMetadata(revisionName: RevisionName) {
+  async restoreMetadata(revisionName: RevisionName) {
     const appmapData = JSON.parse(
       await readFile(this.archivedAppMapFilePath(revisionName, true), 'utf-8')
     );
@@ -208,6 +228,7 @@ export class AppMapReference {
     return {
       sourceLocation: (appmap.metadata as any).source_location,
       appmapName: appmap.metadata.name,
+      testStatus: appmap.metadata.test_status,
       sourcePaths: [...sourcePaths].sort(),
     };
   }

packages/cli/src/describeChange/buildChangeReport.ts

@@ -1,13 +1,15 @@
 import { AppMapReference } from './AppMapReference';
 import { OpenAPIV3 } from 'openapi-types';
-import { Changes, Operation, OperationChange, RouteChanges } from './types';
+import { Changes, LogEntry, Operation, OperationChange, RouteChanges, TestFailure } from './types';
 import assert from 'assert';
 import { executeCommand } from './executeCommand';
 import { exists, shuffleArray } from '../utils';
 import { OperationReference } from './OperationReference';
 import { RevisionName } from './RevisionName';
 import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram';
 import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams';
+import { readFile } from 'fs/promises';
+import { buildAppMap } from '@appland/models';
 
 export default async function buildChangeReport(
   diffDiagrams: DiffDiagrams,
@@ -187,7 +189,43 @@ export default async function buildChangeReport(
     }
   }
 
+  const appmapFileNamesOfFailedTests = await AppMapReference.listFailedTests(
+    operationReference.outputDir,
+    RevisionName.Head
+  );
+
+  const buildFailedTest = async (appmapFileName: string): Promise<TestFailure> => {
+    const appmapData = await readFile(appmapFileName, 'utf-8');
+    const appmap = buildAppMap().source(appmapData).build();
+
+    const logEntries = appmap.events
+      .filter((event) => event.isCall() && event.labels.has('log'))
+      .map((log) => {
+        const message = (log.parameters || []).map((param) => param.value).join('');
+        if (message) {
+          return {
+            message,
+            stack: log
+              .callStack()
+              .map((event) => event.codeObject.location || event.codeObject.fqid),
+          };
+        }
+      })
+      .filter(Boolean) as LogEntry[];
+
+    return {
+      appmapFile: appmapFileName,
+      name: appmap.metadata.name,
+      testLocation: appmap.metadata.source_location,
+      logEntries,
+    };
+  };
+
+  const failedTests = await Promise.all(appmapFileNamesOfFailedTests.map(buildFailedTest));
+
   return {
     routeChanges,
+    findings: [],
+    failedTests,
   };
 }

packages/cli/src/describeChange/types.ts

@@ -18,6 +18,24 @@ export type RouteChanges = {
   changed: OperationChange[];
 };
 
+export type Finding = {
+  ruleId: string;
+};
+
+export type LogEntry = {
+  stack: string[];
+  message: string;
+};
+
+export type TestFailure = {
+  appmapFile: string;
+  name: string;
+  testLocation?: string;
+  logEntries: LogEntry[];
+};
+
 export type Changes = {
   routeChanges: RouteChanges;
+  findings: Finding[];
+  failedTests: TestFailure[];
 };

packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts

@@ -8,19 +8,23 @@ export async function renderSequenceDiagramPNG(
   diagramPath: string,
   browser: Browser
 ): Promise<void> {
-  return new Promise((resolve) =>
+  return new Promise((resolve, reject) =>
     serveAndOpenSequenceDiagram(diagramPath, false, async (url) => {
-      if (verbose()) console.warn(`Rendering PNG`);
-      assert(browser, 'Browser not initialized');
+      try {
+        if (verbose()) console.warn(`Rendering PNG`);
+        assert(browser, 'Browser not initialized');
 
-      const page = await browser.newPage();
-      if (verbose()) page.on('console', (msg) => console.log('CONSOLE: ', msg.text()));
+        const page = await browser.newPage();
+        if (verbose()) page.on('console', (msg) => console.log('CONSOLE: ', msg.text()));
 
-      await page.goto(url);
-      await page.waitForSelector('.sequence-diagram');
-      await page.screenshot({ path: outputPath, fullPage: true });
+        await page.goto(url);
+        await page.waitForSelector('.sequence-diagram', { timeout: 2 * 1000 });
+        await page.screenshot({ path: outputPath, fullPage: true });
 
-      resolve();
+        resolve();
+      } catch (e) {
+        reject(e);
+      }
     })
   );
 }