Link to CI builds

autoblocksai · Apr 12, 2024 · 79a6ff2 · 79a6ff2
1 parent 3417bdd
commit 79a6ff2
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 63 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -61,6 +61,7 @@ jobs:
         working-directory: e2e/python
         env:
           PYTHONPATH: ${{ github.workspace }}/e2e/python
+          FORCE_COLOR: 2
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
 
@@ -106,6 +107,7 @@ jobs:
         run: ../../bin/cli.js testing exec -- npx tsx run.ts
         working-directory: e2e/typescript
         env:
+          FORCE_COLOR: 2
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
 

diff --git a/e2e/python/run.py b/e2e/python/run.py
@@ -72,7 +72,7 @@ def gen_test_cases(n: int) -> list[MyTestCase]:
   run_test_suite(
       id="my-test-suite",
       fn=test_fn,
-      test_cases=gen_test_cases(40),
+      test_cases=gen_test_cases(4),
       evaluators=[
           HasAllSubstrings(),
           IsFriendly(),

diff --git a/e2e/typescript/run.ts b/e2e/typescript/run.ts
@@ -82,7 +82,7 @@ function genTestCases(n: number): MyTestCase[] {
     id: 'my-test-suite',
     fn: testFn,
     testCaseHash: ['input'],
-    testCases: genTestCases(40),
+    testCases: genTestCases(4),
     evaluators: [new HasAllSubstrings(), new IsFriendly()],
   });
 })();
diff --git a/src/handlers/testing/exec/components/progress/index.tsx b/src/handlers/testing/exec/components/progress/index.tsx
@@ -2,16 +2,25 @@ import { Box, Spacer, Static, Text, render } from 'ink';
 import Spinner from 'ink-spinner';
 import { useEffect, useState } from 'react';
 import { EventName, emitter, type EventSchemas } from '../../util/emitter';
-import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../../util/constants';
 import { makeTestRunStatusFromEvaluations } from '../../util/evals';
 import { EvaluationPassed, TestRunStatus } from '../../util/models';
+import {
+  makeAutoblocksCIBuildHtmlUrl,
+  makeAutoblocksTestHtmlUrl,
+} from '../../util/url';
 
 type ConsoleLog = EventSchemas[EventName.CONSOLE_LOG];
 type UncaughtError = EventSchemas[EventName.UNCAUGHT_ERROR];
 type Evaluation = EventSchemas[EventName.EVALUATION];
 type RunStarted = EventSchemas[EventName.RUN_STARTED];
 type RunEnded = EventSchemas[EventName.RUN_ENDED];
 
+interface AppProps {
+  onListenersCreated: () => void;
+  ciBranchId: string | undefined;
+  ciBuildId: string | undefined;
+}
+
 const MAX_TEST_CASES = 100;
 
 /**
@@ -86,6 +95,10 @@ function TestRow(props: {
   const { color: testStatusColor, icon: testStatusIcon } =
     statusToColorAndIcon[testStatus];
 
+  const autoblocksTestHtmlUrl = makeAutoblocksTestHtmlUrl({
+    testExternalId: props.testExternalId,
+  });
+
   return (
     <Box flexDirection="column">
       <Box columnGap={1}>
@@ -95,15 +108,13 @@ function TestRow(props: {
           <Spinner type="dots" />
         )}
         <Text bold={true}>{props.testExternalId}</Text>
-        {/* TODO: show URL for CI context as well */}
-        {!process.env.CI && (
-          <>
-            <Spacer />
-            <Text>
-              {`${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/local/test/${encodeURIComponent(props.testExternalId)}`}
-            </Text>
-          </>
-        )}
+        <Spacer />
+        {/* Hardcode the width so that the URL doesn't wrap. */}
+        <Box width={autoblocksTestHtmlUrl.length}>
+          <Text color="cyan" dimColor={true}>
+            {autoblocksTestHtmlUrl}
+          </Text>
+        </Box>
       </Box>
       <Box paddingLeft={2} columnGap={2}>
         {props.runIsOver && testStatus === TestRunStatus.NO_RESULTS && (
@@ -161,7 +172,7 @@ function TestRow(props: {
   );
 }
 
-const App = (props: { onListenersCreated: () => void }) => {
+const App = (props: AppProps) => {
   const [testExternalIds, setTestExternalIds] = useState<string[]>([]);
   const [consoleLogs, setConsoleLogs] = useState<ConsoleLog[]>([]);
   const [uncaughtErrors, setUncaughtErrors] = useState<UncaughtError[]>([]);
@@ -221,6 +232,14 @@ const App = (props: { onListenersCreated: () => void }) => {
     };
   }, []);
 
+  const autoblocksCIBuildHtmlUrl =
+    props.ciBranchId && props.ciBuildId
+      ? makeAutoblocksCIBuildHtmlUrl({
+          branchId: props.ciBranchId,
+          buildId: props.ciBuildId,
+        })
+      : undefined;
+
   return (
     <>
       <Static items={consoleLogs}>
@@ -247,42 +266,56 @@ const App = (props: { onListenersCreated: () => void }) => {
         // NOTE: This margin is required to prevent the logs being shown in the <Static> component
         // above from clobbering this box
         marginTop={1}
-        paddingX={1}
+        width={'100%'}
         flexDirection="column"
-        borderStyle="round"
-        borderColor="gray"
-        minHeight={12}
-        rowGap={1}
       >
-        {testExternalIds.length === 0 && (
-          <Box>
-            <Text color="gray">
-              Waiting for test results
-              <Spinner type="simpleDots" />
+        {autoblocksCIBuildHtmlUrl && (
+          // Hardcode the width so that the URL doesn't wrap.
+          <Box width={`View results at ${autoblocksCIBuildHtmlUrl}`.length}>
+            <Text color="gray">View results at</Text>
+            <Space />
+            <Text color="cyan" dimColor={true}>
+              {autoblocksCIBuildHtmlUrl}
             </Text>
           </Box>
         )}
-        {testExternalIds.map((testExternalId) => {
-          const testEvals = evals.filter(
-            (e) => e.testExternalId === testExternalId,
-          );
-          const errors = uncaughtErrors.filter(
-            (e) => e.testExternalId === testExternalId,
-          );
-          return (
-            <TestRow
-              key={testExternalId}
-              runIsOver={testIdToRunIsOver[testExternalId]}
-              testExternalId={testExternalId}
-              evals={testEvals}
-              errors={errors}
-            />
-          );
-        })}
+        <Box
+          paddingX={1}
+          flexDirection="column"
+          borderStyle="round"
+          borderColor="gray"
+          minHeight={12}
+          rowGap={1}
+        >
+          {testExternalIds.length === 0 && (
+            <Box>
+              <Text color="gray">
+                Waiting for test results
+                <Spinner type="simpleDots" />
+              </Text>
+            </Box>
+          )}
+          {testExternalIds.map((testExternalId) => {
+            const testEvals = evals.filter(
+              (e) => e.testExternalId === testExternalId,
+            );
+            const testErrors = uncaughtErrors.filter(
+              (e) => e.testExternalId === testExternalId,
+            );
+            return (
+              <TestRow
+                key={testExternalId}
+                runIsOver={testIdToRunIsOver[testExternalId]}
+                testExternalId={testExternalId}
+                evals={testEvals}
+                errors={testErrors}
+              />
+            );
+          })}
+        </Box>
       </Box>
     </>
   );
 };
 
-export const renderTestProgress = (args: { onListenersCreated: () => void }) =>
-  render(<App onListenersCreated={args.onListenersCreated} />);
+export const renderTestProgress = (args: AppProps) => render(<App {...args} />);
diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
@@ -19,20 +19,34 @@ export async function exec(args: {
   exit1OnEvaluationFailure: boolean;
   slackWebhookUrl: string | undefined;
 }) {
+  const runManager = new RunManager({
+    apiKey: args.apiKey,
+    runMessage: args.runMessage,
+  });
+
+  const ciContext = await runManager.setupCIContext();
+
   let listenersCreated = false;
-  renderTestProgress({ onListenersCreated: () => (listenersCreated = true) });
+  renderTestProgress({
+    onListenersCreated: () => (listenersCreated = true),
+    // ciBranchId: runManager.getCIBranchId(),
+    ciBranchId: 'cluww9plh0003129dd72q22bw',
+    // ciBuildId: runManager.getCIBuildId(),
+    ciBuildId: 'cluwxzk5o0005rt80mq1kp0ny',
+  });
 
   // Wait for listeners to be created before starting the server
   while (!listenersCreated) {
     await new Promise((resolve) => setTimeout(resolve, 10));
   }
 
-  const runManager = new RunManager({
-    apiKey: args.apiKey,
-    runMessage: args.runMessage,
-  });
-
-  const ciContext = await runManager.setupCIContext();
+  if (ciContext) {
+    emitter.emit(EventName.CONSOLE_LOG, {
+      ctx: 'cli',
+      level: 'debug',
+      message: `Running in CI environment: ${JSON.stringify(ciContext, null, 2)}`,
+    });
+  }
 
   let runningServer: ServerType | undefined = undefined;
 

diff --git a/src/handlers/testing/exec/util/run-manager.ts b/src/handlers/testing/exec/util/run-manager.ts
@@ -119,12 +119,6 @@ export class RunManager {
       return null;
     }
 
-    emitter.emit(EventName.CONSOLE_LOG, {
-      ctx: 'cli',
-      level: 'debug',
-      message: `Running in CI environment: ${JSON.stringify(ciContext, null, 2)}`,
-    });
-
     const { id, branchId } = await this.post<{ id: string; branchId: string }>(
       '/builds',
       {
@@ -156,6 +150,14 @@ export class RunManager {
     return ciContext;
   }
 
+  getCIBranchId(): string | undefined {
+    return this.ciBranchId;
+  }
+
+  getCIBuildId(): string | undefined {
+    return this.ciBuildId;
+  }
+
   async handleStartRun(args: { testExternalId: string }): Promise<string> {
     emitter.emit(EventName.RUN_STARTED, {
       testExternalId: args.testExternalId,

diff --git a/src/handlers/testing/exec/util/slack.ts b/src/handlers/testing/exec/util/slack.ts
@@ -1,6 +1,7 @@
 import { CIContext } from './ci';
 import { makeTestRunStatusFromEvaluations } from './evals';
 import { Evaluation, EvaluationPassed, TestRunStatus } from './models';
+import { makeAutoblocksCIBuildHtmlUrl } from './url';
 
 // Commit messages are truncated if they're longer than this
 const MAX_COMMIT_MESSAGE_LENGTH = 50;
@@ -56,7 +57,7 @@ function makeSlackMessageBlocks(args: {
             text: 'View in Autoblocks',
             emoji: true,
           },
-          url: makeLinkToBuildInAutoblocks({
+          url: makeAutoblocksCIBuildHtmlUrl({
             branchId: args.branchId,
             buildId: args.buildId,
           }),
@@ -75,13 +76,6 @@ function makeSlackMessageBlocks(args: {
   };
 }
 
-function makeLinkToBuildInAutoblocks(args: {
-  branchId: string;
-  buildId: string;
-}): string {
-  return `https://app.autoblocks.ai/testing/ci?branchId=${args.branchId}&buildId=${args.buildId}`;
-}
-
 function makeDurationString(durationMs: number): string {
   const durationSeconds = durationMs / 1000;
   const durationMinutes = durationSeconds / 60;

diff --git a/src/handlers/testing/exec/util/url.ts b/src/handlers/testing/exec/util/url.ts
@@ -0,0 +1,15 @@
+import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../util/constants';
+
+export function makeAutoblocksCIBuildHtmlUrl(args: {
+  branchId: string;
+  buildId: string;
+}): string {
+  return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/ci?branchId=${args.branchId}&buildId=${args.buildId}`;
+}
+
+export function makeAutoblocksTestHtmlUrl(args: {
+  testExternalId: string;
+}): string {
+  const subpath = process.env.CI ? 'ci' : 'local';
+  return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/${subpath}/test/${encodeURIComponent(args.testExternalId)}`;
+}