chore: multiple measures to speed up e2e tests (#4637)

increase timeout to 10min
cache verdaccio storage in github actions
only write .npmrc during publish
cache packages in verdaccio up to 1 day

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

.github/workflows/e2e-test.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    types: [synchronize, opened, reopened, edited]
     branches: [main]
   schedule:
     # check daily to notice potential package manager issues
@@ -34,6 +33,11 @@ jobs:
           cache: 'npm'
           cache-dependency-path: 'npm-shrinkwrap.json'
           check-latest: true
+      - name: Cache verdaccio storage
+        uses: actions/cache@v3
+        with:
+          path: ./.verdaccio-storage
+          key: verdaccio-e2e-cli-${{ hashFiles('./npm-shrinkwrap.json') }}
       - name: Install dependencies
         run: npm ci --no-audit
       - name: Install pnpm

.gitignore

@@ -28,11 +28,11 @@ site/src/**/*.md
 !site/src/register-addon-thanks.md
 
 # tests
+.verdaccio-storage
 .eslintcache
 tests/integration/hugo-site/resources
 tests/integration/hugo-site/out
 tests/integration/hugo-site/.hugo_build.lock
 _test_out/**
 *.crt
 *.key
-

e2e.config.mjs

@@ -6,7 +6,7 @@ const config = {
   failFast: false,
   failWithoutAssertions: false,
   tap: false,
-  timeout: '5m',
+  timeout: '10m',
 }
 
 export default config

e2e/utils.mjs

@@ -1,9 +1,4 @@
 import { execSync } from 'child_process'
-import { readFileSync } from 'fs'
-import { env } from 'process'
-import { fileURLToPath } from 'url'
-
-const { version } = JSON.parse(readFileSync(fileURLToPath(new URL('../package.json', import.meta.url)), 'utf-8'))
 
 /**
  * Checks if a package manager exists
@@ -21,15 +16,15 @@ export const packageManagerExists = (packageManager) => {
 
 export const packageManagerConfig = {
   npm: {
-    install: ['npm', ['install', 'netlify-cli@testing', `--registry=${env.E2E_TEST_REGISTRY}`]],
+    install: ['npm', ['install', 'netlify-cli@testing']],
     lockFile: 'package-lock.json',
   },
   pnpm: {
-    install: ['pnpm', ['add', `${env.E2E_TEST_REGISTRY}netlify-cli/-/netlify-cli-${version}.tgz`]],
+    install: ['pnpm', ['add', 'netlify-cli@testing']],
     lockFile: 'pnpm-lock.yaml',
   },
   yarn: {
-    install: ['yarn', ['add', 'netlify-cli@testing', `--registry=${env.E2E_TEST_REGISTRY}`]],
+    install: ['yarn', ['add', 'netlify-cli@testing']],
     lockFile: 'yarn.lock',
   },
 }

package.json

@@ -345,7 +345,7 @@
     "tomlify-j0.4": "^3.0.0",
     "tree-kill": "^1.2.2",
     "typescript": "^4.4.4",
-    "verdaccio": "^5.4.0"
+    "verdaccio": "^5.10.2"
   },
   "ava": {
     "files": [

tools/e2e/setup.mjs

@@ -1,7 +1,8 @@
 import { appendFileSync, existsSync, promises, readFileSync, writeFileSync } from 'fs'
-import { homedir, tmpdir } from 'os'
+import { tmpdir } from 'os'
 import { join, sep } from 'path'
 import { cwd, env } from 'process'
+import { fileURLToPath } from 'url'
 
 import del from 'del'
 import execa from 'execa'
@@ -35,6 +36,8 @@ const getVerdaccioConfig = (storage) => ({
   uplinks: {
     npmjs: {
       url: 'https://registry.npmjs.org/',
+      maxage: '1d',
+      cache: true,
     },
   },
   packages: {
@@ -64,7 +67,11 @@ export const startRegistry = async () => {
   // number in parallel
   const startPort = Math.floor(Math.random() * END_PORT_RANGE) + START_PORT_RANGE
   const freePort = await getPort({ host: 'localhost', port: startPort })
-  const storage = await mkdtemp(`${tmpdir()}${sep}verdaccio-`)
+  const storage = fileURLToPath(new URL('../../.verdaccio-storage', import.meta.url))
+
+  // Remove netlify-cli from the verdaccio storage because we are going to publish it in a second
+  await rmdirRecursiveAsync(join(storage, 'netlify-cli'))
+
   return new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error('Starting Verdaccio Timed out'))
@@ -91,23 +98,18 @@ export const setup = async () => {
   const { storage, url } = await startRegistry()
   const workspace = await mkdtemp(`${tmpdir()}${sep}e2e-test-`)
 
-  const npmrc = join(homedir(), '.npmrc')
+  const npmrc = fileURLToPath(new URL('../../.npmrc', import.meta.url))
   const registryWithAuth = `//${url.hostname}:${url.port}/:_authToken=dummy`
   let backupNpmrc
 
   /** Cleans up everything */
   const cleanup = async () => {
-    // restore ~/.npmrc
-    if (backupNpmrc) {
-      writeFileSync(npmrc, backupNpmrc)
-    } else {
-      await rmdirRecursiveAsync(npmrc)
-    }
     // remote temp folders
-    await rmdirRecursiveAsync(storage)
     await rmdirRecursiveAsync(workspace)
   }
 
+  env.npm_config_registry = url
+
   try {
     if (existsSync(npmrc)) {
       backupNpmrc = readFileSync(npmrc, 'utf-8')
@@ -126,16 +128,20 @@ export const setup = async () => {
   Verdaccio: ${storage}
   Workspace: ${workspace}
 ------------------------------------------`)
-
-    writeFileSync(join(workspace, '.npmrc'), registryWithAuth, 'utf-8')
   } catch (error_) {
     await cleanup()
     throw new Error(
       `npm publish failed for registry ${url.href}
-Be sure not to have a ~/.npmrc in your home folder that specifies a different registry.
 
 ${error_ instanceof Error ? error_.message : error_}`,
     )
+  } finally {
+    // restore .npmrc
+    if (backupNpmrc) {
+      writeFileSync(npmrc, backupNpmrc)
+    } else {
+      await rmdirRecursiveAsync(npmrc)
+    }
   }
 
   return {