EPD-621 Updates for CI

.github/workflows/e2e.yml

@@ -0,0 +1,67 @@
+name: E2E
+
+on:
+ # Run on different types of events to ensure we are
+ # handling the git data correctly in each scenario
+ push:
+ pull_request:
+ schedule:
+ - cron: '17 15 * * *'
+
+env:
+ AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+ TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
+
+jobs:
+ py:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install CLI dependencies
+ run: npm ci
+
+ - name: Build CLI
+ run: npm run build
+
+ - name: Install dependencies in e2e/python
+ run: pip install -r requirements.txt
+
+ - name: Run tests in e2e/python
+ run: ../../bin/cli.js testing exec -- python3 run.py
+ working-directory: e2e/python
+ env:
+ PYTHONPATH: ${{ github.workspace }}/e2e/python
+
+ ts:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install CLI dependencies
+ run: npm ci
+
+ - name: Build CLI
+ run: npm run build
+
+ - name: Install dependencies in e2e/typescript
+ run: npm install
+ working-directory: e2e/typescript
+
+ - name: Run tests in e2e/typescript
+ run: ../../bin/cli.js testing exec -- npx tsx run.ts
+ working-directory: e2e/typescript

.prettierignore

@@ -2,3 +2,4 @@
 *.py
 CODEOWNERS
 *.sh
+*.txt

e2e/python/requirements.txt

@@ -0,0 +1 @@
+autoblocksai

e2e/python/run.py

@@ -0,0 +1,80 @@
+import uuid
+import random
+import asyncio
+import dataclasses
+
+from autoblocks.testing.models import BaseTestCase
+from autoblocks.testing.models import BaseTestEvaluator
+from autoblocks.testing.models import Evaluation
+from autoblocks.testing.models import Threshold
+from autoblocks.testing.util import md5
+from autoblocks.testing.run import run_test_suite
+
+
+@dataclasses.dataclass
+class MyTestCase(BaseTestCase):
+ input: str
+ expected_substrings: list[str]
+
+ def hash(self) -> str:
+ return md5(self.input)
+
+
+async def test_fn(test_case: MyTestCase) -> str:
+ await asyncio.sleep(random.random())
+
+ substrings = test_case.input.split("-")
+ if random.random() < 0.2:
+ substrings.pop()
+
+ return "-".join(substrings)
+
+
+class HasAllSubstrings(BaseTestEvaluator):
+ id = "has-all-substrings"
+
+ def evaluate_test_case(self, test_case: MyTestCase, output: str) -> Evaluation:
+ score = 1 if all(s in output for s in test_case.expected_substrings) else 0
+ return Evaluation(
+ score=score,
+ threshold=Threshold(gte=1),
+ )
+
+
+class IsFriendly(BaseTestEvaluator):
+ id = "is-friendly"
+
+ async def get_score(self, output: str) -> float:
+ await asyncio.sleep(random.random())
+ return random.random()
+
+ async def evaluate_test_case(self, test_case: BaseTestCase, output: str) -> Evaluation:
+ score = await self.get_score(output)
+ return Evaluation(
+ score=score,
+ )
+
+
+def gen_test_cases(n: int) -> list[MyTestCase]:
+ test_cases = []
+ for _ in range(n):
+ random_id = str(uuid.uuid4())
+ test_cases.append(
+ MyTestCase(
+ input=random_id,
+ expected_substrings=random_id.split("-"),
+ ),
+ )
+ return test_cases
+
+
+if __name__ == "__main__":
+ run_test_suite(
+ id="my-test-suite",
+ fn=test_fn,
+ test_cases=gen_test_cases(40),
+ evaluators=[
+ HasAllSubstrings(),
+ IsFriendly(),
+ ],
+ )

e2e/typescript/package.json

@@ -0,0 +1,9 @@
+{
+ "name": "ts-e2e",
+ "version": "0.0.0",
+ "private": true,
+ "dependencies": {
+ "@autoblocks/client": "*",
+ "typescript": "*"
+ }
+}

e2e/typescript/run.ts

@@ -0,0 +1,88 @@
+import {
+ BaseTestEvaluator,
+ runTestSuite,
+ type Evaluation,
+} from '@autoblocks/client/testing';
+import * as crypto from 'crypto';
+
+interface MyTestCase {
+ input: string;
+ expectedSubstrings: string[];
+}
+
+async function testFn({ testCase }: { testCase: MyTestCase }): Promise<string> {
+ // Simulate doing work
+ await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+
+ const substrings = testCase.input.split('-');
+ if (Math.random() < 0.2) {
+ // Remove a substring randomly. This will cause about 20% of the test cases to fail
+ // the "has-all-substrings" evaluator.
+ substrings.pop();
+ }
+
+ return substrings.join('-');
+}
+
+class HasAllSubstrings extends BaseTestEvaluator<MyTestCase, string> {
+ id = 'has-all-substrings';
+
+ evaluateTestCase(args: { testCase: MyTestCase; output: string }): Evaluation {
+ const score = args.testCase.expectedSubstrings.every((s) =>
+ args.output.includes(s),
+ )
+ ? 1
+ : 0;
+
+ return {
+ score,
+ threshold: {
+ gte: 1,
+ },
+ };
+ }
+}
+
+class IsFriendly extends BaseTestEvaluator<MyTestCase, string> {
+ id = 'is-friendly';
+
+ async getScore(output: string): Promise<number> {
+ // eslint-disable-next-line no-console
+ console.log(`Determining score for output: ${output}`);
+ await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+ return Math.random();
+ }
+
+ async evaluateTestCase(args: {
+ testCase: MyTestCase;
+ output: string;
+ }): Promise<Evaluation> {
+ const score = await this.getScore(args.output);
+
+ return {
+ score,
+ };
+ }
+}
+
+function genTestCases(n: number): MyTestCase[] {
+ const testCases: MyTestCase[] = [];
+ for (let i = 0; i < n; i++) {
+ const randomId = crypto.randomUUID();
+ testCases.push({
+ input: randomId,
+ expectedSubstrings: randomId.split('-'),
+ });
+ }
+ return testCases;
+}
+
+(async () => {
+ await runTestSuite<MyTestCase, string>({
+ id: 'my-test-suite',
+ fn: testFn,
+ testCaseHash: ['input'],
+ testCases: genTestCases(40),
+ evaluators: [new HasAllSubstrings(), new IsFriendly()],
+ });
+})();

e2e/typescript/tsconfig.json

@@ -0,0 +1,8 @@
+{
+ "compilerOptions": {
+ "moduleResolution": "node",
+ "module": "esnext",
+ "target": "esnext"
+ },
+ "include": ["**/*.ts"]
+}

package.json

@@ -30,7 +30,7 @@
  "homepage": "https://github.com/autoblocksai/cli#readme",
  "dependencies": {
  "@actions/exec": "^1.1.1",
- "@autoblocks/client": "^0.0.32",
+ "@autoblocks/client": "^0.0.33",
  "@hono/node-server": "^1.7.0",
  "@hono/zod-validator": "^0.1.11",
  "hono": "^3.12.12",

src/handlers/testing/exec/emitter.ts

@@ -12,7 +12,7 @@ export enum EventName {
 
 // Zod schemas for event data
 const zConsoleLogSchema = z.object({
- ctx: z.enum(['cmd', 'cli', 'cli-server']),
+ ctx: z.enum(['cmd', 'cli']),
  level: z.enum(['debug', 'info', 'warn', 'error']),
  message: z.string(),
 });