chore: print stdout/stderr when dev server fails in tests (#4918)

* chore: wait in tests till edge functions server is running and print stdout/stderr on failure

* chore: instead of logging add to error

this way if the error is catched we do not print anything

* chore: only log message in DEBUG or tests

* chore: fix tests

* chore: remove useless change and use setImmediate to alleviate this a little

* chore: typo

tests/integration/600.framework-detection.test.js

@@ -12,9 +12,6 @@ const { normalize } = require('./utils/snapshots')
 
 const content = 'Hello World!'
 
-// Remove extra '.'s resulting from different load times which lead to flaky snapshots
-const frameworkDetectionNormalizer = (output) => normalize(output.replace(/\.+(?=\.)/, ''))
-
 const test = isCI ? avaTest.serial.bind(avaTest) : avaTest
 
 test('should default to process.cwd() and static server', async (t) => {
@@ -30,7 +27,7 @@ test('should default to process.cwd() and static server', async (t) => {
       const response = await got(url).text()
       t.is(response, content)
 
-      t.snapshot(frameworkDetectionNormalizer(output))
+      t.snapshot(normalize(output))
     })
   })
 })
@@ -48,7 +45,7 @@ test('should use static server when --dir flag is passed', async (t) => {
       const response = await got(url).text()
       t.is(response, content)
 
-      t.snapshot(frameworkDetectionNormalizer(output))
+      t.snapshot(normalize(output))
     })
   })
 })
@@ -67,7 +64,7 @@ test('should use static server when framework is set to #static', async (t) => {
       const response = await got(url).text()
       t.is(response, content)
 
-      t.snapshot(frameworkDetectionNormalizer(output))
+      t.snapshot(normalize(output))
     })
   })
 })
@@ -87,7 +84,7 @@ test('should log the command if using static server and `command` is configured'
         const response = await got(url).text()
         t.is(response, content)
 
-        t.snapshot(frameworkDetectionNormalizer(output))
+        t.snapshot(normalize(output))
       },
     )
   })
@@ -108,7 +105,7 @@ test('should warn if using static server and `targetPort` is configured', async
         const response = await got(url).text()
         t.is(response, content)
 
-        t.snapshot(frameworkDetectionNormalizer(output))
+        t.snapshot(normalize(output))
       },
     )
   })
@@ -126,7 +123,7 @@ test('should run `command` when both `command` and `targetPort` are configured',
         true,
       ),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -136,7 +133,7 @@ test('should force a specific framework when configured', async (t) => {
 
     // a failure is expected since this is not a true create-react-app project
     const error = await t.throwsAsync(() => withDevServer({ cwd: builder.directory }, () => {}, true))
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -145,7 +142,7 @@ test('should throw when forcing a non supported framework', async (t) => {
     await builder.withNetlifyToml({ config: { dev: { framework: 'to-infinity-and-beyond-js' } } }).buildAsync()
 
     const error = await t.throwsAsync(() => withDevServer({ cwd: builder.directory }, () => {}, true))
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -159,7 +156,7 @@ test('should detect a known framework', async (t) => {
 
     // a failure is expected since this is not a true create-react-app project
     const error = await t.throwsAsync(() => withDevServer({ cwd: builder.directory }, () => {}, true))
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -170,7 +167,7 @@ test('should throw if framework=#custom but command is missing', async (t) => {
     const error = await t.throwsAsync(() =>
       withDevServer({ cwd: builder.directory, args: ['--targetPort', '3000'] }, () => {}, true),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -181,7 +178,7 @@ test('should throw if framework=#custom but targetPort is missing', async (t) =>
     const error = await t.throwsAsync(() =>
       withDevServer({ cwd: builder.directory, args: ['--command', 'echo hello'] }, () => {}, true),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -196,7 +193,7 @@ test('should start custom command if framework=#custom, command and targetPort a
         true,
       ),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -221,7 +218,7 @@ test(`should print specific error when command doesn't exist`, async (t) => {
         true,
       ),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -250,7 +247,7 @@ test('should prompt when multiple frameworks are detected', async (t) => {
 
       await childProcess
     })
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -266,7 +263,7 @@ test('should not run framework detection if command and targetPort are configure
         true,
       ),
     )
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 
@@ -286,7 +283,7 @@ test('should filter frameworks with no dev command', async (t) => {
       const response = await got(url).text()
       t.is(response, content)
 
-      t.snapshot(frameworkDetectionNormalizer(output))
+      t.snapshot(normalize(output))
     })
   })
 })
@@ -305,7 +302,7 @@ test('should pass framework-info env to framework sub process', async (t) => {
 
     // a failure is expected since this is not a true Gatsby project
     const error = await t.throwsAsync(() => withDevServer({ cwd: builder.directory }, () => {}, true))
-    t.snapshot(frameworkDetectionNormalizer(error.stdout))
+    t.snapshot(normalize(error.stdout))
   })
 })
 

tests/integration/utils/dev-server.js

@@ -24,7 +24,9 @@ const startServer = async ({ cwd, offline = true, env = {}, args = [], expectFai
   const staticPort = await getPort()
   const host = 'localhost'
   const url = `http://${host}:${port}`
+
   console.log(`Starting dev server on port: ${port} in directory ${path.basename(cwd)}`)
+
   const ps = execa(
     cliPath,
     ['dev', offline ? '--offline' : '', '-p', port, '--staticServerPort', staticPort, ...args],
@@ -38,23 +40,37 @@ const startServer = async ({ cwd, offline = true, env = {}, args = [], expectFai
   }
 
   const outputBuffer = []
+  const errorBuffer = []
   const serverPromise = new Promise((resolve, reject) => {
     let selfKilled = false
+    ps.stderr.on('data', (data) => {
+      errorBuffer.push(data)
+    })
     ps.stdout.on('data', (data) => {
       outputBuffer.push(data)
       if (!expectFailure && data.includes('Server now ready on')) {
-        resolve({
-          url,
-          host,
-          port,
-          output: outputBuffer.join(''),
-          outputBuffer,
-          close: async () => {
-            selfKilled = true
-            await killProcess(ps)
-          },
-          promptHistory,
-        })
+        setImmediate(() =>
+          resolve({
+            url,
+            host,
+            port,
+            errorBuffer,
+            outputBuffer,
+            get output() {
+              // these are getters so we do the actual joining as late as possible as the array might still get
+              // populated after we resolve here
+              return outputBuffer.join('')
+            },
+            get error() {
+              return errorBuffer.join('')
+            },
+            close: async () => {
+              selfKilled = true
+              await killProcess(ps)
+            },
+            promptHistory,
+          }),
+        )
       }
     })
     // eslint-disable-next-line promise/prefer-await-to-callbacks,promise/prefer-await-to-then
@@ -91,6 +107,12 @@ const withDevServer = async (options, testHandler, expectFailure = false) => {
   try {
     server = await startDevServer(options, expectFailure)
     return await testHandler(server)
+  } catch (error) {
+    if (server && !expectFailure) {
+      error.stdout = server.output
+      error.stderr = server.error
+    }
+    throw error
   } finally {
     if (server) {
       await server.close()