Add hard retrying for Azure test runs

azure-pipelines.yml

@@ -35,5 +35,5 @@ steps:
     displayName: 'Yarn check'
 
   - script: |
-      node run-tests.js -c 2 -g $(group)
+      node run-tests.js -c 2 -g $(group) --hard-retry
     displayName: 'Run tests'

package.json

@@ -106,6 +106,7 @@
     "node-sass": "4.12.0",
     "pre-commit": "1.2.2",
     "prettier": "1.17.1",
+    "ps-list": "6.3.0",
     "react": "16.8.6",
     "react-dom": "16.8.6",
     "react-ssr-prepass": "1.0.5",

run-tests.js

@@ -1,16 +1,21 @@
 const path = require('path')
 const _glob = require('glob')
+const psList = require('ps-list')
+const _treeKill = require('tree-kill')
 const { promisify } = require('util')
 const { Sema } = require('async-sema')
 const { spawn, exec: execOrig } = require('child_process')
 
 const glob = promisify(_glob)
 const exec = promisify(execOrig)
+const treeKill = promisify(_treeKill)
 
 const NUM_RETRIES = 2
 const DEFAULT_CONCURRENCY = 3
 
 ;(async () => {
+  // kills all node process except current one and all Chrome(driver) instances
+  const useHardRetries = process.argv.indexOf('--hard-retry') > -1
   let concurrencyIdx = process.argv.indexOf('-c')
   const concurrency =
     parseInt(process.argv[concurrencyIdx + 1], 10) || DEFAULT_CONCURRENCY
@@ -68,12 +73,20 @@ const DEFAULT_CONCURRENCY = 3
           stdio: 'inherit'
         }
       )
-      children.add(child)
-      child.on('exit', code => {
+      const exitHandler = code => {
         children.delete(child)
         if (code) reject(new Error(`failed with code: ${code}`))
         resolve()
-      })
+      }
+      child.on('exit', exitHandler)
+
+      child.prepareRestart = () => {
+        child.removeListener('exit', exitHandler)
+        children.delete(child)
+        child.kill()
+      }
+      child.restart = () => resolve(runTest(test))
+      children.add(child)
     })
 
   await Promise.all(
@@ -93,6 +106,24 @@ const DEFAULT_CONCURRENCY = 3
               await exec(`git clean -fdx "${path.join(__dirname, test)}"`)
               await exec(`git checkout "${path.join(__dirname, test)}"`)
             } catch (err) {}
+
+            if (useHardRetries) {
+              const runningChildren = [...children]
+              runningChildren.forEach(child => child.prepareRestart())
+              for (const proc of await psList()) {
+                const name = proc.name.toLowerCase()
+
+                if (name.includes('chrome') || name.includes('node')) {
+                  if (proc.pid !== process.pid) {
+                    console.log('killing', name)
+                    try {
+                      await treeKill(proc.pid)
+                    } catch (_) {}
+                  }
+                }
+              }
+              runningChildren.forEach(child => child.restart())
+            }
           }
         }
       }

yarn.lock

@@ -11410,6 +11410,11 @@ prr@~1.0.1:
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
   integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY=
 
+ps-list@6.3.0:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/ps-list/-/ps-list-6.3.0.tgz#a2b775c2db7d547a28fbaa3a05e4c281771259be"
+  integrity sha512-qau0czUSB0fzSlBOQt0bo+I2v6R+xiQdj78e1BR/Qjfl5OHWJ/urXi8+ilw1eHe+5hSeDI1wrwVTgDp2wst4oA==
+
 ps-tree@=1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ps-tree/-/ps-tree-1.1.1.tgz#5f1ba35455b8c25eeb718d04c37de1555d96d3db"