Delete me 2024 09 20 testing ci

.github/workflows/efps.yml

@@ -0,0 +1,81 @@
+name: eFPS Test
+on:
+  pull_request:
+jobs:
+  install:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Store Playwright's Version
+        run: |
+          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
+          echo "Playwright's Version: $PLAYWRIGHT_VERSION"
+          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
+
+      - name: Cache Playwright Browsers for Playwright's Version
+        id: cache-playwright-browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
+
+      - name: Install Playwright Browsers
+        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+
+      - name: Run eFPS tests
+        env:
+          VITE_PERF_EFPS_PROJECT_ID: ${{ secrets.PERF_EFPS_PROJECT_ID }}
+          VITE_PERF_EFPS_DATASET: ${{ secrets.PERF_EFPS_DATASET }}
+          PERF_EFPS_SANITY_TOKEN: ${{ secrets.PERF_EFPS_SANITY_TOKEN }}
+        run: pnpm efps:test
+
+      - name: PR comment with report
+        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          comment_tag: "efps-report"
+          filePath: ${{ github.workspace }}/perf/efps/results/benchmark-results.md
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: efps-report
+          path: perf/efps/results
+          retention-days: 30

package.json

@@ -54,6 +54,7 @@
     "e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup",
     "e2e:start": "pnpm --filter studio-e2e-testing preview",
     "etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl",
+    "efps:test": "cd perf/efps && pnpm test",
     "example:blog-studio": "cd examples/blog-studio && pnpm start",
     "example:clean-studio": "cd examples/blog-studio && pnpm start",
     "example:ecommerce-studio": "cd examples/blog-studio && pnpm start",

packages/sanity/src/core/form/store/utils/createMemoizer.ts

@@ -28,7 +28,7 @@ export function createMemoizer<TFunction extends (...args: never[]) => unknown>(
       const hashed = hashInput(...args)
       const serializedHash = JSON.stringify(hashed)
       const cached = cache.get(path)
-      if (serializedHash === cached?.serializedHash) return cached.result
+      // if (serializedHash === cached?.serializedHash) return cached.result
 
       const result = fn(...args) as ReturnType<TFunction>
       cache.set(path, {serializedHash, result})

perf/efps/index.ts

@@ -1,7 +1,10 @@
+/* eslint-disable max-depth */
 /* eslint-disable no-console */
 // eslint-disable-next-line import/no-unassigned-import
 import 'dotenv/config'
 
+import fs from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 import process from 'node:process'
 import {fileURLToPath} from 'node:url'
@@ -11,13 +14,13 @@ import chalk from 'chalk'
 import Table from 'cli-table3'
 import Ora from 'ora'
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import {exec} from './helpers/exec'
 import {runTest} from './runTest'
 import article from './tests/article/article'
 import recipe from './tests/recipe/recipe'
 import singleString from './tests/singleString/singleString'
 import synthetic from './tests/synthetic/synthetic'
+import {type EfpsResult} from './types'
 
 const headless = true
 const tests = [singleString, recipe, article, synthetic]
@@ -63,29 +66,87 @@ await exec({
   cwd: monorepoRoot,
 })
 
+// Prepare the latest version of the 'sanity' package
+const tmpDir = path.join(os.tmpdir(), `sanity-latest-${Date.now()}`)
+await fs.promises.mkdir(tmpDir, {recursive: true})
+spinner.start('')
+await exec({
+  command: 'npm install sanity@latest --no-save',
+  cwd: tmpDir,
+  spinner,
+  text: ['Downloading latest sanity package…', 'Downloaded latest sanity package'],
+})
+const sanityPackagePath = path.join(tmpDir, 'node_modules', 'sanity')
+
 await exec({
   text: ['Ensuring playwright is installed…', 'Playwright is installed'],
   command: 'npx playwright install',
   spinner,
 })
 
 const table = new Table({
-  head: [chalk.bold('benchmark'), 'eFPS p50', 'eFPS p75', 'eFPS p90'].map((cell) =>
-    chalk.cyan(cell),
+  head: [chalk.bold('benchmark'), 'Passed?', 'p50 eFPS (Δ%)', 'p75 eFPS (Δ%)', 'p90 eFPS (Δ%)'].map(
+    (cell) => chalk.cyan(cell),
   ),
 })
 
+const markdownRows: string[] = []
+
 const formatFps = (fps: number) => {
   const rounded = fps.toFixed(1)
   if (fps >= 60) return chalk.green(rounded)
   if (fps < 20) return chalk.red(rounded)
   return chalk.yellow(rounded)
 }
 
+const formatPercentage = (value: number): string => {
+  const rounded = value.toFixed(1)
+  const sign = value >= 0 ? '+' : ''
+  if (value > -50) return `${sign}${rounded}%`
+  return chalk.red(`${sign}${rounded}%`)
+}
+
+// For markdown formatting without colors
+const formatFpsPlain = (fps: number) => {
+  const rounded = fps.toFixed(1)
+  return rounded
+}
+
+const formatPercentagePlain = (value: number): string => {
+  const rounded = value.toFixed(1)
+  const sign = value >= 0 ? '+' : ''
+  return `${sign}${rounded}%`
+}
+
+function getStatus(p50Diff: number, p75Diff: number, p90Diff: number): 'error' | 'passed' {
+  if (p50Diff < -50 || p75Diff < -50 || p90Diff < -50) {
+    return 'error'
+  }
+  return 'passed'
+}
+
+function getStatusEmoji(status: 'error' | 'passed'): string {
+  if (status === 'error') return '🔴'
+  return '✅'
+}
+
+// Initialize the overall status
+let overallStatus: 'error' | 'passed' = 'passed'
+
+interface TestResult {
+  testName: string
+  version: 'local' | 'latest'
+  results: EfpsResult[]
+}
+
+const allResults: TestResult[] = []
+
 for (let i = 0; i < tests.length; i++) {
   const test = tests[i]
-  const results = await runTest({
-    prefix: `Running '${test.name}' [${i + 1}/${tests.length}]…`,
+
+  // Run with local 'sanity' package
+  const localResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with local 'sanity'…`,
     test,
     resultsDir,
     spinner,
@@ -94,14 +155,86 @@ for (let i = 0; i < tests.length; i++) {
     projectId,
   })
 
-  for (const result of results) {
-    table.push({
-      [[chalk.bold(test.name), result.label ? `(${result.label})` : ''].join(' ')]: [
-        formatFps(result.p50),
-        formatFps(result.p75),
-        formatFps(result.p90),
-      ],
-    })
+  allResults.push({
+    testName: test.name,
+    version: 'local',
+    results: localResults,
+  })
+
+  // Run with latest 'sanity' package
+  const latestResults = await runTest({
+    prefix: `Running '${test.name}' [${i + 1}/${tests.length}] with 'sanity@latest'…`,
+    test,
+    resultsDir,
+    spinner,
+    client,
+    headless,
+    projectId,
+    sanityPackagePath,
+  })
+
+  allResults.push({
+    testName: test.name,
+    version: 'latest',
+    results: latestResults,
+  })
+}
+
+for (const test of tests) {
+  const localResult = allResults.find((r) => r.testName === test.name && r.version === 'local')
+  const latestResult = allResults.find((r) => r.testName === test.name && r.version === 'latest')
+
+  if (localResult && latestResult) {
+    const localResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of localResult.results) {
+      localResultsMap.set(res.label, res)
+    }
+    const latestResultsMap = new Map<string | undefined, EfpsResult>()
+    for (const res of latestResult.results) {
+      latestResultsMap.set(res.label, res)
+    }
+
+    for (const [label, latest] of latestResultsMap) {
+      const local = localResultsMap.get(label)
+      if (local) {
+        // Compute percentage differences
+        const p50Diff = ((local.p50 - latest.p50) / latest.p50) * 100
+        const p75Diff = ((local.p75 - latest.p75) / latest.p75) * 100
+        const p90Diff = ((local.p90 - latest.p90) / latest.p90) * 100
+
+        // Determine test status
+        const testStatus = getStatus(p50Diff, p75Diff, p90Diff)
+
+        // Update overall status
+        if (testStatus === 'error') {
+          overallStatus = 'error'
+        }
+
+        const rowLabel = [chalk.bold(test.name), label ? `(${label})` : ''].join(' ')
+
+        table.push([
+          rowLabel,
+          getStatusEmoji(testStatus),
+          `${formatFps(local.p50)} (${formatPercentage(p50Diff)})`,
+          `${formatFps(local.p75)} (${formatPercentage(p75Diff)})`,
+          `${formatFps(local.p90)} (${formatPercentage(p90Diff)})`,
+        ])
+
+        // Add to markdown rows
+        const markdownRow = [
+          [test.name, label ? `(${label})` : ''].join(' '),
+          getStatusEmoji(testStatus),
+          `${formatFpsPlain(local.p50)} (${formatPercentagePlain(p50Diff)})`,
+          `${formatFpsPlain(local.p75)} (${formatPercentagePlain(p75Diff)})`,
+          `${formatFpsPlain(local.p90)} (${formatPercentagePlain(p90Diff)})`,
+        ]
+        markdownRows.push(`| ${markdownRow.join(' | ')} |`)
+      } else {
+        spinner.fail(`Missing local result for test '${test.name}', label '${label}'`)
+      }
+    }
+  } else {
+    spinner.fail(`Missing results for test '${test.name}'`)
   }
 }
 
@@ -113,3 +246,35 @@ console.log(`
 │ The number of renders ("frames") that is assumed to be possible
 │ within a second. Derived from input latency. ${chalk.green('Higher')} is better.
 `)
+
+// Map overallStatus to status text
+const statusText = overallStatus === 'error' ? 'Error' : 'Passed'
+const statusEmoji = getStatusEmoji(overallStatus)
+
+// Build the markdown content
+const markdownContent = [
+  '# Benchmark Results',
+  '',
+  `<details>`,
+  `<summary>${statusEmoji} Performance Benchmark Results — Status: **${statusText}** </summary>`,
+  '',
+  '| Benchmark | Passed? | p50 eFPS (Δ%) | p75 eFPS (Δ%) | p90 eFPS (Δ%) |',
+  '|-----------|---------|---------------|---------------|---------------|',
+  ...markdownRows,
+  '</details>',
+  '',
+  '> **eFPS — editor "Frames Per Second"**',
+  '> ',
+  '> The number of renders ("frames") that is assumed to be possible within a second. Derived from input latency. **Higher** is better.',
+  '',
+].join('\n')
+
+// Write markdown file to root of results
+const markdownOutputPath = path.join(workspaceDir, 'results', 'benchmark-results.md')
+await fs.promises.writeFile(markdownOutputPath, markdownContent)
+
+// Exit with code 1 if regression detected
+if (overallStatus === 'error') {
+  console.error(chalk.red('Performance regression detected exceeding 50% threshold.'))
+  process.exit(1)
+}