feat(auth): playwright tests (#4570)

* feat(auth): playwright tests

* add test server

* add workflow

* update server

* test workflow

* playwright workflow

* playwright workflow

* WIP

* WIP

.github/workflows/playwright.yaml

@@ -0,0 +1,44 @@
+name: Playwright Tests
+
+env:
+  pip-version: 24.2
+
+on:
+  push:
+    branches: [auth, main]
+  pull_request:
+    branches: [auth, main]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+      - name: Install dependencies
+        working-directory: ./app
+        run: npm install -g pnpm && pnpm install
+      - name: Install Playwright Browsers
+        working-directory: ./app
+        run: pnpm exec playwright install --with-deps
+      - name: Build the app
+        working-directory: ./app
+        run: pnpm run build
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install Phoenix
+        run: |
+          python -m pip install --upgrade pip==${{ env.pip-version }}
+          pip install .
+      - name: Run Playwright tests
+        working-directory: ./app
+        run: pnpm exec playwright test
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: app/playwright-report/
+          retention-days: 30

app/.gitignore

@@ -1,3 +1,7 @@
 node_modules
 dist
-stats.html
+stats.html
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

app/package.json

@@ -50,11 +50,13 @@
   },
   "devDependencies": {
     "@emotion/react": "^11.11.4",
+    "@playwright/test": "^1.47.0",
     "@types/d3-format": "^3.0.4",
     "@types/d3-scale-chromatic": "^3.0.3",
     "@types/d3-time-format": "^4.0.3",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.17.7",
+    "@types/node": "^22.5.4",
     "@types/react": "18.2.48",
     "@types/react-dom": "^18.3.0",
     "@types/react-relay": "^16.0.6",
@@ -88,11 +90,14 @@
     "build:static": "cpy ./static ../src/phoenix/server",
     "build:relay": "relay-compiler",
     "test": "jest --config ./jest.config.js",
+    "test:e2e": "playwright test",
+    "test:e2e:ui": "playwright test --ui",
     "dev": "pnpm run dev:server & pnpm run build:static && pnpm run build:relay && vite",
     "dev:auth": "export PHOENIX_ENABLE_AUTH=true && export PHOENIX_SECRET=sa18f44bd9b6b4b0606e58a2d03d09039aee8283e0074c6517fda58577a07dd#A && pnpm run dev:server & pnpm run build:static && pnpm run build:relay && vite",
     "dev:ui": "pnpm run build:static && pnpm run build:relay && vite",
-    "dev:server:init": "python -m phoenix.server.main --dev serve --with-fixture=chatbot --with-project=demo_llama_index --force-fixture-ingestion",
     "dev:server": "python -m phoenix.server.main --dev serve",
+    "dev:server:test": "node ./tests/utils/testServer.mjs",
+    "dev:server:init": "python -m phoenix.server.main --dev serve --with-fixture=chatbot --with-project=demo_llama_index --force-fixture-ingestion",
     "typecheck": "tsc --noEmit",
     "lint": "eslint ./src",
     "prettier": "prettier --write './src/**/*'",

app/playwright.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./tests",
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:6006",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "setup",
+      testMatch: /global\.setup\.ts/,
+    },
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"] },
+    },
+    {
+      name: "webkit",
+      use: { ...devices["Desktop Safari"] },
+    },
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: "pnpm run dev:server:test",
+    url: "http://localhost:6006",
+    reuseExistingServer: false, // !process.env.CI,
+  },
+});