diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e92d758..956a045 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -61,6 +61,7 @@ jobs: working-directory: e2e/python env: PYTHONPATH: ${{ github.workspace }}/e2e/python + FORCE_COLOR: 2 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }} @@ -106,6 +107,7 @@ jobs: run: ../../bin/cli.js testing exec -- npx tsx run.ts working-directory: e2e/typescript env: + FORCE_COLOR: 2 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }} diff --git a/e2e/python/run.py b/e2e/python/run.py index 24959f9..f39ad12 100644 --- a/e2e/python/run.py +++ b/e2e/python/run.py @@ -72,7 +72,7 @@ def gen_test_cases(n: int) -> list[MyTestCase]: run_test_suite( id="my-test-suite", fn=test_fn, - test_cases=gen_test_cases(40), + test_cases=gen_test_cases(4), evaluators=[ HasAllSubstrings(), IsFriendly(), diff --git a/e2e/typescript/run.ts b/e2e/typescript/run.ts index ed10494..209e783 100644 --- a/e2e/typescript/run.ts +++ b/e2e/typescript/run.ts @@ -82,7 +82,7 @@ function genTestCases(n: number): MyTestCase[] { id: 'my-test-suite', fn: testFn, testCaseHash: ['input'], - testCases: genTestCases(40), + testCases: genTestCases(4), evaluators: [new HasAllSubstrings(), new IsFriendly()], }); })(); diff --git a/src/handlers/testing/exec/components/progress/index.tsx b/src/handlers/testing/exec/components/progress/index.tsx index 015ddff..6f4906a 100644 --- a/src/handlers/testing/exec/components/progress/index.tsx +++ b/src/handlers/testing/exec/components/progress/index.tsx @@ -2,9 +2,12 @@ import { Box, Spacer, Static, Text, render } from 'ink'; import Spinner from 'ink-spinner'; import { useEffect, useState } from 'react'; import { EventName, emitter, type EventSchemas } from '../../util/emitter'; -import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../../util/constants'; import { makeTestRunStatusFromEvaluations } from '../../util/evals'; import { EvaluationPassed, TestRunStatus } from '../../util/models'; +import { + makeAutoblocksCIBuildHtmlUrl, + makeAutoblocksTestHtmlUrl, +} from '../../util/url'; type ConsoleLog = EventSchemas[EventName.CONSOLE_LOG]; type UncaughtError = EventSchemas[EventName.UNCAUGHT_ERROR]; @@ -12,6 +15,12 @@ type Evaluation = EventSchemas[EventName.EVALUATION]; type RunStarted = EventSchemas[EventName.RUN_STARTED]; type RunEnded = EventSchemas[EventName.RUN_ENDED]; +interface AppProps { + onListenersCreated: () => void; + ciBranchId: string | undefined; + ciBuildId: string | undefined; +} + const MAX_TEST_CASES = 100; /** @@ -86,6 +95,10 @@ function TestRow(props: { const { color: testStatusColor, icon: testStatusIcon } = statusToColorAndIcon[testStatus]; + const autoblocksTestHtmlUrl = makeAutoblocksTestHtmlUrl({ + testExternalId: props.testExternalId, + }); + return ( @@ -95,15 +108,13 @@ function TestRow(props: { )} {props.testExternalId} - {/* TODO: show URL for CI context as well */} - {!process.env.CI && ( - <> - - - {`${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/local/test/${encodeURIComponent(props.testExternalId)}`} - - - )} + + {/* Hardcode the width so that the URL doesn't wrap. */} + + + {autoblocksTestHtmlUrl} + + {props.runIsOver && testStatus === TestRunStatus.NO_RESULTS && ( @@ -161,7 +172,7 @@ function TestRow(props: { ); } -const App = (props: { onListenersCreated: () => void }) => { +const App = (props: AppProps) => { const [testExternalIds, setTestExternalIds] = useState([]); const [consoleLogs, setConsoleLogs] = useState([]); const [uncaughtErrors, setUncaughtErrors] = useState([]); @@ -221,6 +232,14 @@ const App = (props: { onListenersCreated: () => void }) => { }; }, []); + const autoblocksCIBuildHtmlUrl = + props.ciBranchId && props.ciBuildId + ? makeAutoblocksCIBuildHtmlUrl({ + branchId: props.ciBranchId, + buildId: props.ciBuildId, + }) + : undefined; + return ( <> @@ -247,42 +266,56 @@ const App = (props: { onListenersCreated: () => void }) => { // NOTE: This margin is required to prevent the logs being shown in the component // above from clobbering this box marginTop={1} - paddingX={1} + width="100%" flexDirection="column" - borderStyle="round" - borderColor="gray" - minHeight={12} - rowGap={1} > - {testExternalIds.length === 0 && ( - - - Waiting for test results - + {autoblocksCIBuildHtmlUrl && ( + // Hardcode the width so that the URL doesn't wrap. + + View results at + + + {autoblocksCIBuildHtmlUrl} )} - {testExternalIds.map((testExternalId) => { - const testEvals = evals.filter( - (e) => e.testExternalId === testExternalId, - ); - const errors = uncaughtErrors.filter( - (e) => e.testExternalId === testExternalId, - ); - return ( - - ); - })} + + {testExternalIds.length === 0 && ( + + + Waiting for test results + + + + )} + {testExternalIds.map((testExternalId) => { + const testEvals = evals.filter( + (e) => e.testExternalId === testExternalId, + ); + const testErrors = uncaughtErrors.filter( + (e) => e.testExternalId === testExternalId, + ); + return ( + + ); + })} + ); }; -export const renderTestProgress = (args: { onListenersCreated: () => void }) => - render(); +export const renderTestProgress = (args: AppProps) => render(); diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts index fb6895c..c4bf2ba 100644 --- a/src/handlers/testing/exec/index.ts +++ b/src/handlers/testing/exec/index.ts @@ -19,20 +19,32 @@ export async function exec(args: { exit1OnEvaluationFailure: boolean; slackWebhookUrl: string | undefined; }) { + const runManager = new RunManager({ + apiKey: args.apiKey, + runMessage: args.runMessage, + }); + + const ciContext = await runManager.setupCIContext(); + let listenersCreated = false; - renderTestProgress({ onListenersCreated: () => (listenersCreated = true) }); + renderTestProgress({ + onListenersCreated: () => (listenersCreated = true), + ciBranchId: runManager.getCIBranchId(), + ciBuildId: runManager.getCIBuildId(), + }); // Wait for listeners to be created before starting the server while (!listenersCreated) { await new Promise((resolve) => setTimeout(resolve, 10)); } - const runManager = new RunManager({ - apiKey: args.apiKey, - runMessage: args.runMessage, - }); - - const ciContext = await runManager.setupCIContext(); + if (ciContext) { + emitter.emit(EventName.CONSOLE_LOG, { + ctx: 'cli', + level: 'debug', + message: `Running in CI environment: ${JSON.stringify(ciContext, null, 2)}`, + }); + } let runningServer: ServerType | undefined = undefined; diff --git a/src/handlers/testing/exec/util/run-manager.ts b/src/handlers/testing/exec/util/run-manager.ts index cb07f57..4cabcaa 100644 --- a/src/handlers/testing/exec/util/run-manager.ts +++ b/src/handlers/testing/exec/util/run-manager.ts @@ -119,12 +119,6 @@ export class RunManager { return null; } - emitter.emit(EventName.CONSOLE_LOG, { - ctx: 'cli', - level: 'debug', - message: `Running in CI environment: ${JSON.stringify(ciContext, null, 2)}`, - }); - const { id, branchId } = await this.post<{ id: string; branchId: string }>( '/builds', { @@ -156,6 +150,14 @@ export class RunManager { return ciContext; } + getCIBranchId(): string | undefined { + return this.ciBranchId; + } + + getCIBuildId(): string | undefined { + return this.ciBuildId; + } + async handleStartRun(args: { testExternalId: string }): Promise { emitter.emit(EventName.RUN_STARTED, { testExternalId: args.testExternalId, diff --git a/src/handlers/testing/exec/util/slack.ts b/src/handlers/testing/exec/util/slack.ts index db32ee8..ced94bf 100644 --- a/src/handlers/testing/exec/util/slack.ts +++ b/src/handlers/testing/exec/util/slack.ts @@ -1,6 +1,7 @@ import { CIContext } from './ci'; import { makeTestRunStatusFromEvaluations } from './evals'; import { Evaluation, EvaluationPassed, TestRunStatus } from './models'; +import { makeAutoblocksCIBuildHtmlUrl } from './url'; // Commit messages are truncated if they're longer than this const MAX_COMMIT_MESSAGE_LENGTH = 50; @@ -56,7 +57,7 @@ function makeSlackMessageBlocks(args: { text: 'View in Autoblocks', emoji: true, }, - url: makeLinkToBuildInAutoblocks({ + url: makeAutoblocksCIBuildHtmlUrl({ branchId: args.branchId, buildId: args.buildId, }), @@ -75,13 +76,6 @@ function makeSlackMessageBlocks(args: { }; } -function makeLinkToBuildInAutoblocks(args: { - branchId: string; - buildId: string; -}): string { - return `https://app.autoblocks.ai/testing/ci?branchId=${args.branchId}&buildId=${args.buildId}`; -} - function makeDurationString(durationMs: number): string { const durationSeconds = durationMs / 1000; const durationMinutes = durationSeconds / 60; diff --git a/src/handlers/testing/exec/util/url.ts b/src/handlers/testing/exec/util/url.ts new file mode 100644 index 0000000..7e23ee5 --- /dev/null +++ b/src/handlers/testing/exec/util/url.ts @@ -0,0 +1,15 @@ +import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../util/constants'; + +export function makeAutoblocksCIBuildHtmlUrl(args: { + branchId: string; + buildId: string; +}): string { + return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/ci?branchId=${args.branchId}&buildId=${args.buildId}`; +} + +export function makeAutoblocksTestHtmlUrl(args: { + testExternalId: string; +}): string { + const subpath = process.env.CI ? 'ci' : 'local'; + return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/${subpath}/test/${encodeURIComponent(args.testExternalId)}`; +}