Debugging Tests

.github/workflows/test.yaml

@@ -590,15 +590,14 @@ jobs:
         with:
           node-version: 20
 
-      - name: Setup Bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: latest
-
       - uses: pnpm/action-setup@v2
         with:
           version: latest
 
+      - name: Install Dependencies
+        run: pnpm i
+        working-directory: examples/ci
+
       - name: Deploy
         run: |
           pnpm add @upstash/workflow@${{needs.release.outputs.version}}
@@ -610,7 +609,7 @@ jobs:
         working-directory: examples/ci
 
       - name: Test
-        run: bun test .
+        run: pnpm test
         working-directory: examples/ci
 
   cloudflare-workers-hono-deployed:

examples/ci/app/ci/ci.test.ts

@@ -1,10 +1,13 @@
-import { test, describe } from "bun:test"
+import { test, describe } from "vitest";
 import { TEST_ROUTES, TEST_TIMEOUT_DURATION } from "./constants";
 import { initiateTest } from "./utils";
+import { config } from "dotenv";
+
+config();
 
 describe("workflow integration tests", () => {
   TEST_ROUTES.forEach(testConfig => {
     test(
       testConfig.route,
       async () => {
         await initiateTest(testConfig.route)

examples/ci/app/ci/constants.ts

@@ -1,6 +1,6 @@
 import { TestConfig } from "./types"
 
-export const RETRY_COUNT = 20
+export const RETRY_COUNT = 30
 export const RETRY_INTERVAL_DURATION = 2000
 export const CHECK_WF_AFTER_INIT_DURATION = 10000
 const TEST_BUFFER_DURATION = 5000

examples/ci/app/ci/upstash/qstash.ts

@@ -2,7 +2,7 @@ import { Client, QstashError } from "@upstash/qstash"
 import { TEST_ROUTE_PREFIX, CI_RANDOM_ID_HEADER, CI_ROUTE_HEADER } from "../constants"
 import { TestConfig } from "../types"
 
-const client = new Client({ baseUrl: process.env.QSTASH_URL, token: process.env.QSTASH_TOKEN! })
+const client = new Client({ baseUrl: process.env.QSTASH_URL, token: process.env.QSTASH_TOKEN!, retry: false })
 
 /**
  * starts a workflow run given the config
@@ -22,7 +22,7 @@ export const startWorkflow = async (
       [ CI_ROUTE_HEADER ]: testConfig.route,
       ...testConfig.headers
     },
-    body: testConfig.payload
+    body: testConfig.payload,
   })
   return result
 }

examples/ci/app/ci/upstash/redis.test.ts

@@ -1,21 +1,18 @@
-
-import { describe, test, expect, beforeAll } from "bun:test";
+import { describe, test, expect, beforeAll } from "vitest";
 import * as redis from "./redis"
 import { nanoid } from "../utils";
 
 describe("redis", () => {
-  test("should throw on missing results", () => {
-    expect(() =>
-      redis.checkRedisForResults("some-route", "some-id", -1, "some-result", 1)
-    ).toThrow(
-      "result not found for route some-route with randomTestId some-id"
+  test("should throw on missing results", { timeout: 15000 },async () => {
+    await expect(redis.checkRedisForResults("some-route", "some-id", -1, "some-result", 1)).rejects.toThrowError(
+            "result not found for route some-route with randomTestId some-id"
     )
-  }, { timeout: 15000 })
+  })
 
   test("should throw when saving results without any increment", () => {
-    expect(() =>
-      redis.saveResultsWithoutContext("some-route", "some-id", "some-result")
-    ).toThrow(
+    expect(async () =>
+      await redis.saveResultsWithoutContext("some-route", "some-id", "some-result")
+    ).rejects.toThrowError(
       "callCount shouldn't be 0. It was 0 in test of the route 'some-route'"
     )
   })
@@ -34,52 +31,53 @@ describe("redis", () => {
 
     test("should throw on mismatching result", () => {
       expect(async () =>
-        redis.checkRedisForResults(route, randomId, 2, "not-correct")
-      ).toThrow(
-        `Unexpected value.\n\tReceived "${result}"\n\tExpected "not-correct"`
+        await redis.checkRedisForResults(route, randomId, 2, "not-correct")
+      ).rejects.toThrowError(
+        `Unexpected value.\n\tReceived \"${result}\"\n\tExpected \"not-correct\"`
       )
     })
 
     test("should throw on mismatching call count", () => {
       expect(async () =>
-        redis.checkRedisForResults(route, randomId, 123, result)
-      ).toThrow(
-        `Unexpected value.\n\tReceived "2"\n\tExpected "123"`
+        await redis.checkRedisForResults(route, randomId, 123, result)
+      ).rejects.toThrowError(
+        `Unexpected value.\n\tReceived \"2\"\n\tExpected \"123\"`
       )
     })
 
     test("should not throw on correct results", () => {
       expect(async () =>
-        redis.checkRedisForResults(route, randomId, 2, result)
-      ).not.toThrow()
+        await redis.checkRedisForResults(route, randomId, 2, result)
+      ).not.toThrowError()
     })
   })
 
-  test("should override call count", async () => {
+  describe("override call count", () => {
+    test("should override call count", async () => {
+      const route = "override-route";
+      const randomId = `random-id-${nanoid()}`;
+      const result = `random-result-${nanoid()}`;
+      const override = -3;
 
-    const route = "override-route"
-    const randomId = `random-id-${nanoid()}`
-    const result = `random-result-${nanoid()}`
-    const override = -3
-
-    await redis.increment(route, randomId)
-    await redis.increment(route, randomId)
-    await redis.increment(route, randomId)
+      await redis.increment(route, randomId);
+      await redis.increment(route, randomId);
+      await redis.increment(route, randomId);
 
-    await redis.saveResultsWithoutContext(route, randomId, result, override)
+      await redis.saveResultsWithoutContext(route, randomId, result, override);
 
-    expect(async () =>
-      redis.checkRedisForResults(route, randomId, 3, result)
-    ).toThrow(
-      `Unexpected value.\n\tReceived "-3"\n\tExpected "3"`
-    )
+      expect(async () =>
+        await redis.checkRedisForResults(route, randomId, 3, result)
+      ).rejects.toThrowError(
+        `Unexpected value.\n\tReceived \"-3\"\n\tExpected \"3\"`
+      );
+    });
 
     test("should not throw on correct results", () => {
       expect(async () =>
-        redis.checkRedisForResults(route, randomId, override, result)
-      ).not.toThrow()
-    })
-  })
+        await redis.checkRedisForResults("override-route", `random-id-${nanoid()}`, -3, `random-result-${nanoid()}`)
+      ).not.toThrowError();
+    });
+  });
 
   test("should fail if marked as failed", async () => {
 
@@ -94,7 +92,7 @@ describe("redis", () => {
 
     // mark as failed and check
     await redis.failWithoutContext(route, randomId)
-    expect(redis.checkRedisForResults(route, randomId, 1, result)).rejects.toThrow(redis.FAILED_TEXT)
+    expect(redis.checkRedisForResults(route, randomId, 1, result)).rejects.toThrowError(redis.FAILED_TEXT)
 
   })
 })

examples/ci/app/ci/upstash/redis.ts

@@ -120,6 +120,8 @@ export const checkRedisForResults = async (
 
   const retryCount = retryOverride ?? RETRY_COUNT
   for ( let i=1; i <= retryCount; i++ ) {
+    console.log(`check ${i}`);
+
     testResult = await redis.get<RedisResult>(key)
     if (testResult) {
       break

examples/ci/app/ci/utils.test.ts

@@ -1,7 +1,7 @@
-import { test, expect } from "bun:test";
+import { test, expect } from "vitest";
 import { expect as utilsExpect } from "./utils"
 
 test("expect", () => {
-  expect(() => utilsExpect("same", "same")).not.toThrow()
-  expect(() => utilsExpect("not", "same")).toThrow(`Unexpected value.\n\tReceived "not"\n\tExpected "same"`)
+  expect(() => utilsExpect("same", "same")).not.toThrowError()
+  expect(() => utilsExpect("not", "same")).toThrowError()
 })

examples/ci/app/ci/utils.ts

@@ -64,22 +64,28 @@ export const getTestConfig = async (route: string) => {
 
 export const initiateTest = async (route: string) => {
   const randomTestId = nanoid()
+  console.log(`initiating test for ${route}`);
+
   const { headers, payload, expectedCallCount, expectedResult, shouldWorkflowStart = true } = await getTestConfig(route)
+  console.log(`got config`);
 
   const { messageId } = await qstash.startWorkflow({ route, headers, payload }, randomTestId)
+  console.log(`starting workflow`);
 
   // sleep for 4 secs and check that message is delivered
   await new Promise(r => setTimeout(r, CHECK_WF_AFTER_INIT_DURATION));
 
   try {
     await qstash.checkWorkflowStart(messageId);
+    console.log(`workflow started`);
   } catch (error) {
     console.error(error);
     if (shouldWorkflowStart) {
       throw error;
     };
   }
 
+  console.log("checking redis for results");
   await redis.checkRedisForResults(route, randomTestId, expectedCallCount, expectedResult)
 }
 

examples/ci/app/test-routes/wait-for-event/notifier/route.ts

@@ -16,7 +16,7 @@ export const POST = async (request: Request) => {
 
   const sleepAndNotify = async () => {
 
-    await new Promise(r => setTimeout(r, 1000));
+    await new Promise(r => setTimeout(r, 3000));
 
     const result = await client.notify({
       eventId: sdkEventId,

examples/ci/app/test-routes/wait-for-event/workflow/route.ts

@@ -54,7 +54,7 @@ export const { POST, GET } = testServe(
           }
         ),
         context.waitForEvent("wait sdk", config.sdkEventId, {
-          timeout: "5s"
+          timeout: "10s"
         })
       ])
 

examples/ci/package.json

@@ -6,14 +6,16 @@
     "dev": "next dev -p 3001",
     "build": "next build",
     "start": "next start -p 3001",
-    "lint": "next lint"
+    "lint": "next lint",
+    "test": "vitest"
   },
   "dependencies": {
     "@upstash/qstash": "^2.7.12",
     "@upstash/redis": "^1.34.2",
     "@upstash/workflow": "latest",
     "@vercel/functions": "^1.5.0",
     "clsx": "^2.1.1",
+    "dotenv": "^16.5.0",
     "next": "14.2.23",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
@@ -30,6 +32,7 @@
     "prettier": "^3.3.3",
     "prettier-plugin-tailwindcss": "^0.6.6",
     "tailwindcss": "^3.4.1",
-    "typescript": "^5.6.2"
+    "typescript": "^5.6.2",
+    "vitest": "^3.1.3"
   }
 }

examples/ci/vitest.config.js

@@ -0,0 +1,15 @@
+import { config } from "dotenv";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    passWithNoTests: true,
+    globals: true,
+    coverage: {
+      provider: "v8",
+    },
+    env: {
+      ...config({ path: "./.env" }).parsed,
+    },
+  },
+});

src/client/index.test.ts

@@ -133,10 +133,17 @@ describe("workflow client", () => {
           url: "http://requestcatcher.com",
         });
 
-        const firstCancel = await liveClient.cancel({
-          ids: [workflowRunIdOne, workflowRunIdTwo, "non-existent"],
-        });
-        expect(firstCancel).toEqual({ cancelled: 2 });
+        await new Promise((r) => setTimeout(r, 1000));
+        await eventually(
+          async () => {
+            const firstCancel = await liveClient.cancel({
+              ids: [workflowRunIdOne, workflowRunIdTwo, "non-existent"],
+            });
+            expect(firstCancel).toEqual({ cancelled: 2 });
+          },
+          { timeout: 5000, interval: 1000 }
+        );
+        await new Promise((r) => setTimeout(r, 1000));
 
         // trying to cancel the workflows one by one gives error, as they were canceled above
         const secondCancel = await liveClient.cancel({ ids: workflowRunIdOne });
@@ -147,7 +154,7 @@ describe("workflow client", () => {
         expect(thirdCancel).toEqual({ cancelled: 0 });
       },
       {
-        timeout: 10000,
+        timeout: 20000,
       }
     );
 
@@ -161,14 +168,20 @@ describe("workflow client", () => {
           url: "http://requestcatcher.com/second",
         });
 
-        const cancel = await liveClient.cancel({
-          urlStartingWith: "http://requestcatcher.com",
-        });
+        await new Promise((r) => setTimeout(r, 5000));
 
-        expect(cancel).toEqual({ cancelled: 2 });
+        await eventually(
+          async () => {
+            const cancel = await liveClient.cancel({
+              urlStartingWith: "http://requestcatcher.com",
+            });
+            expect(cancel).toEqual({ cancelled: 2 });
+          },
+          { timeout: 5000, interval: 1000 }
+        );
       },
       {
-        timeout: 10000,
+        timeout: 15000,
       }
     );
 

src/constants.ts

@@ -16,7 +16,7 @@ export const NO_CONCURRENCY = 1;
 export const NOT_SET = "not-set";
 export const DEFAULT_RETRIES = 3;
 
-export const VERSION = "v0.2.7";
+export const VERSION = "v0.2.13";
 export const SDK_TELEMETRY = `@upstash/workflow@${VERSION}`;
 
 export const TELEMETRY_HEADER_SDK = "Upstash-Telemetry-Sdk" as const;

src/serve/serve-many.test.ts

@@ -160,7 +160,7 @@ describe("serveMany", () => {
               "Upstash-Forward-Upstash-Workflow-Sdk-Version": ["1"],
               "Upstash-Telemetry-Framework": ["nextjs"],
               "Upstash-Telemetry-Runtime": ["node@v22.6.0"],
-              "Upstash-Telemetry-Sdk": ["@upstash/workflow@v0.2.7"],
+              "Upstash-Telemetry-Sdk": ["@upstash/workflow@v0.2.13"],
               "Upstash-Workflow-Init": ["false"],
               "Upstash-Workflow-RunId": ["wfr_id"],
               "Upstash-Workflow-Runid": ["wfr_id"],
@@ -215,7 +215,7 @@ describe("serveMany", () => {
               "Upstash-Forward-Upstash-Workflow-Sdk-Version": ["1"],
               "Upstash-Telemetry-Framework": ["nextjs"],
               "Upstash-Telemetry-Runtime": ["node@v22.6.0"],
-              "Upstash-Telemetry-Sdk": ["@upstash/workflow@v0.2.7"],
+              "Upstash-Telemetry-Sdk": ["@upstash/workflow@v0.2.13"],
               "Upstash-Workflow-Init": ["false"],
               "Upstash-Workflow-RunId": ["wfr_id"],
               "Upstash-Workflow-Runid": ["wfr_id"],
@@ -289,7 +289,7 @@ describe("serveMany", () => {
                 "upstash-retries": "0",
                 "upstash-telemetry-framework": "nextjs",
                 "upstash-telemetry-runtime": "node@v22.6.0",
-                "upstash-telemetry-sdk": "@upstash/workflow@v0.2.7",
+                "upstash-telemetry-sdk": "@upstash/workflow@v0.2.13",
                 "upstash-workflow-calltype": "toCallback",
                 "upstash-workflow-init": "false",
                 "upstash-workflow-runid": "wfr_id",