chore(build): Avoid prebuilding api side, instead use an esbuild plug…

…in (redwoodjs#9767)

Resurrecting this again, to see if I can get CI more stable this time 

**2nd edition Implementation:** With the second edition, I'm trying
esbuild's watcher - rather than chokidar
This is a no go ❌, because when new files are introduced the watcher
isn't triggered.

**3rd edition implementation**: Use esbuild's `rebuild` function that
_may_ provide performance benefits

Old PR redwoodjs#7672

---------

Co-authored-by: Tobbe Lundberg <tobbe@tlundberg.com>
Co-authored-by: Dominic Saadi <dominiceliassaadi@gmail.com>

.github/workflows/ci.yml

@@ -179,54 +179,55 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
-      - name: 📁 Create a temporary directory
-        id: createpath
-        run: |
-          project_path=$(mktemp -d -t redwood.XXXXXX)
-          echo "::set-output name=project_path::$project_path"
-          framework_path=$(pwd)
-          echo "::set-output name=framework_path::$framework_path"
+      - name: 🔨 Build
+        run: yarn build
+
+      - name: 🌲 Install Cypress
+        run: yarn cypress install
 
       - name: 🌲 Create a Redwood App
-        run: |
-          ./tasks/run-e2e ${{ steps.createpath.outputs.project_path }} \
-            --no-start \
-            --bundler ${{ matrix.bundler }}
+        id: crwa
         env:
           YARN_ENABLE_IMMUTABLE_INSTALLS: false
-
-      - name: 🐙 Git init in the Redwood App directory
         run: |
+          project_path=$(mktemp -d -t redwood.XXXXXX)
+          echo "project-path=$project_path" >> $GITHUB_OUTPUT
+
           git config --global user.email "you@example.com"
           git config --global user.name "Your Name"
-          git init --initial-branch main && git add .
-          git commit -a --message=init
-        working-directory: ${{ steps.createpath.outputs.project_path }}
 
-      - name: Start server in background
-        run: yarn rw dev --no-generate --fwd="--no-open" &
-        working-directory: ${{ steps.createpath.outputs.project_path }}
+          ./tasks/run-e2e "$project_path" \
+            --bundler ${{ matrix.bundler }} \
+            --no-build-framework \
+            --no-start
 
-      - name: 🌲 Install Cypress
-        run: yarn run cypress install
+      - name: Start the dev server in the background
+        run: |
+          yarn rw dev --no-generate --fwd="--no-open" 2>&1 | tee dev_server.log &
+        working-directory: ${{ steps.crwa.outputs.project-path }}
 
-      - name: 🌲 Run cypress
-        uses: cypress-io/github-action@v5
+      - name: 🌲 Run Cypress
+        uses: cypress-io/github-action@v6
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_RW_PATH:  "${{ steps.createpath.outputs.project_path }}"
+          CYPRESS_RW_PATH: ${{ steps.crwa.outputs.project-path }}
         with:
-          # We've already installed dependencies.
-          install: false
-          env: true
           browser: chrome
-          record: false
+          env: true
+          install: false
           wait-on: 'http://[::1]:8910'
           working-directory: ./tasks/e2e
           spec: |
              cypress/e2e/01-tutorial/*.cy.js
              cypress/e2e/04-logger/*.cy.js
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ matrix.bundler }}-logs
+          path: |
+            ${{ steps.crwa.outputs.project-path }}/dev_server.log
+            ${{ steps.crwa.outputs.project-path }}/e2e.log
+
   tutorial-e2e-skip:
     needs: detect-changes
     if: needs.detect-changes.outputs.onlydocs == 'true'

package.json

@@ -84,6 +84,7 @@
     "boxen": "5.1.2",
     "core-js": "3.34.0",
     "cypress": "13.6.1",
+    "cypress-fail-fast": "7.1.0",
     "cypress-wait-until": "3.0.1",
     "dependency-cruiser": "15.5.0",
     "dotenv": "16.3.1",

packages/api-server/src/watch.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
-import { fork } from 'child_process'
 import type { ChildProcess } from 'child_process'
+import { fork } from 'child_process'
 import fs from 'fs'
 import path from 'path'
 
@@ -13,12 +13,16 @@ import { debounce } from 'lodash'
 import { hideBin } from 'yargs/helpers'
 import yargs from 'yargs/yargs'
 
-import { buildApi } from '@redwoodjs/internal/dist/build/api'
+import {
+  buildApi,
+  cleanApiBuild,
+  rebuildApi,
+} from '@redwoodjs/internal/dist/build/api'
 import { loadAndValidateSdls } from '@redwoodjs/internal/dist/validateSchema'
 import {
-  getPaths,
   ensurePosixPath,
   getConfig,
+  getPaths,
   resolveFile,
 } from '@redwoodjs/project-config'
 
@@ -43,9 +47,6 @@ dotenv.config({
   path: rwjsPaths.base,
 })
 
-// TODO:
-// 1. Move this file out of the HTTP server, and place it in the CLI?
-
 let httpServerProcess: ChildProcess
 
 const killApiServer = () => {
@@ -63,19 +64,32 @@ const validate = async () => {
     console.error(c.red(e?.message))
     console.log(c.redBright('-'.repeat(40)))
 
-    delayRestartServer.cancel()
+    debouncedBuild.cancel()
+    debouncedRebuild.cancel()
     return false
   }
 }
 
-const rebuildApiServer = () => {
+const buildAndRestart = async ({
+  rebuild = false,
+  clean = false,
+}: { rebuild?: boolean; clean?: boolean } = {}) => {
   try {
     // Shutdown API server
     killApiServer()
 
     const buildTs = Date.now()
     process.stdout.write(c.dim(c.italic('Building... ')))
-    buildApi()
+
+    if (clean) {
+      await cleanApiBuild()
+    }
+
+    if (rebuild) {
+      await rebuildApi()
+    } else {
+      await buildApi()
+    }
     console.log(c.dim(c.italic('Took ' + (Date.now() - buildTs) + ' ms')))
 
     const forkOpts = {
@@ -148,11 +162,18 @@ const rebuildApiServer = () => {
 // this usually happens when running RedwoodJS generator commands.
 // Local writes are very fast, but writes in e2e environments are not,
 // so allow the default to be adjust with a env-var.
-const delayRestartServer = debounce(
-  rebuildApiServer,
+const debouncedRebuild = debounce(
+  () => buildAndRestart({ rebuild: true }),
   process.env.RWJS_DELAY_RESTART
     ? parseInt(process.env.RWJS_DELAY_RESTART, 10)
-    : 5
+    : 500
+)
+
+const debouncedBuild = debounce(
+  () => buildAndRestart({ rebuild: false }),
+  process.env.RWJS_DELAY_RESTART
+    ? parseInt(process.env.RWJS_DELAY_RESTART, 10)
+    : 500
 )
 
 // NOTE: the file comes through as a unix path, even on windows
@@ -165,7 +186,7 @@ const IGNORED_API_PATHS = [
 ].map((path) => ensurePosixPath(path))
 
 chokidar
-  .watch(rwjsPaths.api.base, {
+  .watch([rwjsPaths.api.src], {
     persistent: true,
     ignoreInitial: true,
     ignored: (file: string) => {
@@ -188,7 +209,11 @@ chokidar
     },
   })
   .on('ready', async () => {
-    rebuildApiServer()
+    // First time
+    await buildAndRestart({
+      clean: true,
+      rebuild: false,
+    })
     await validate()
   })
   .on('all', async (eventName, filePath) => {
@@ -199,20 +224,30 @@ chokidar
       return
     }
 
-    // We validate here, so that developers will see the error
-    // As they're running the dev server
-    if (filePath.includes('.sdl')) {
-      const isValid = await validate()
+    if (eventName) {
+      if (filePath.includes('.sdl')) {
+        // We validate here, so that developers will see the error
+        // As they're running the dev server
+        const isValid = await validate()
 
-      // Exit early if not valid
-      if (!isValid) {
-        return
+        // Exit early if not valid
+        if (!isValid) {
+          return
+        }
       }
     }
 
     console.log(
       c.dim(`[${eventName}] ${filePath.replace(rwjsPaths.api.base, '')}`)
     )
-    delayRestartServer.cancel()
-    delayRestartServer()
+
+    if (eventName === 'add' || eventName === 'unlink') {
+      debouncedBuild.cancel()
+      debouncedRebuild.cancel()
+      debouncedBuild()
+    } else {
+      // If files have just changed, then rebuild
+      debouncedRebuild.cancel()
+      debouncedRebuild()
+    }
   })

packages/babel-config/jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-  testPathIgnorePatterns: ['fixtures'],
+  testPathIgnorePatterns: ['fixtures', 'dist/*'],
 }

packages/babel-config/package.json

@@ -47,6 +47,7 @@
   "devDependencies": {
     "@types/babel-plugin-tester": "9.0.9",
     "@types/babel__core": "7.20.4",
+    "@types/node": "20.10.4",
     "babel-plugin-tester": "11.0.4",
     "esbuild": "0.19.9",
     "jest": "29.7.0"

packages/babel-config/src/__tests__/api.test.ts

@@ -1,7 +1,8 @@
 import { vol } from 'memfs'
 
-import { getPaths, ensurePosixPath } from '@redwoodjs/project-config'
+import { ensurePosixPath, getPaths } from '@redwoodjs/project-config'
 
+import type { PluginList } from '../api'
 import {
   getApiSideBabelConfigPath,
   getApiSideBabelPlugins,
@@ -87,7 +88,7 @@ describe('api', () => {
       )
 
       const apiSideBabelConfigPath = getApiSideBabelConfigPath()
-      expect(ensurePosixPath(apiSideBabelConfigPath)).toMatch(
+      expect(ensurePosixPath(apiSideBabelConfigPath || '')).toMatch(
         '/redwood-app/api/babel.config.js'
       )
     })
@@ -185,9 +186,17 @@ describe('api', () => {
         },
       ])
 
+      type ModuleResolverConfig = {
+        root: string[]
+        alias: Record<string, string>
+        cwd: string
+        loglevel: string
+      }
+
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       const [_, babelPluginModuleResolverConfig] = apiSideBabelPlugins.find(
         (plugin) => plugin[0] === 'babel-plugin-module-resolver'
-      )
+      )! as [any, ModuleResolverConfig, any]
 
       expect(babelPluginModuleResolverConfig).toMatchObject({
         alias: {
@@ -238,12 +247,12 @@ describe('api', () => {
   })
 })
 
-function getPluginAliases(plugins) {
+function getPluginAliases(plugins: PluginList) {
   return plugins.reduce((pluginAliases, plugin) => {
     if (plugin.length !== 3) {
       return pluginAliases
     }
 
     return [...pluginAliases, plugin[2]]
-  }, [])
+  }, [] as any)
 }