ci: Only run E2E test apps that are affected on PRs (#14254)

This PR aims to solve two things:

1. Do not require us to keep a list of test-applications to run on CI.
This is prone to be forgotten, and then certain tests do not run on CI
and we may not notice (this has happened more than once in the past 😬 )
2. Do not run E2E test applications that are unrelated to anything that
was changed.

With this PR, instead of keeping a list of E2E test jobs we want to run
manually in the build.yml file, this is now inferred (on CI) by a script
based on the nx dependency graph and some config in the test apps
package.json. This is how it works:

1. Pick all folders in test-applications, by default all of them should
run. --> This will "fix" the problem we had sometimes that we forgot to
add test apps to the build.yml so they don't actually run on CI
:grimacing:
3. For each test app, look at it's dependencies (and devDependencies),
and see if any of them has changed in the current PR (based on nx
affected projects). So e.g. if you change something in browser, only E2E
test apps that have any dependency that uses browser will run, so e.g.
node-express will be skipped.
4. Additionally, you can configure variants in the test apps
package.json - instead of defining this in the workflow file, it has to
be defined in the test app itself now.
5. Finally, a test app can be marked as optional in the package.json,
and can also have optional variants to run.
6. The E2E test builds the required & optional matrix on the fly based
on these things.
7. In pushes (=on develop/release branches) we just run everything.
8. If anything inside of the e2e-tests package changed, run everything.
--> This could be further optimized to only run changed test apps, but
this was the easy solution for now.

## Example test runs

* In a PR with no changes related to the E2E tests, nothing is run:
https://github.com/getsentry/sentry-javascript/actions/runs/11838604965
* In a CI run for a push, all E2E tests are run:
https://github.com/getsentry/sentry-javascript/actions/runs/11835834748
* In a PR with a change in e2e-tests itself, all E2E tests are run:
https://github.com/getsentry/sentry-javascript/actions/runs/11836668035
* In a PR with a change in the node package, only related E2E tests are
run:
https://github.com/getsentry/sentry-javascript/actions/runs/11838274483

Author: mydea

.github/workflows/build.yml

@@ -801,7 +801,15 @@ jobs:
     needs: [job_get_metadata, job_build, job_compile_bindings_profiling_node]
     runs-on: ubuntu-20.04-large-js
     timeout-minutes: 15
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+      matrix-optional: ${{ steps.matrix-optional.outputs.matrix }}
     steps:
+      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
+        uses: actions/checkout@v4
+        if: github.event_name == 'pull_request'
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -851,11 +859,21 @@ jobs:
           path: ${{ github.workspace }}/packages/*/*.tgz
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
+      - name: Determine which E2E test applications should be run
+        id: matrix
+        run: yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
+      - name: Determine which optional E2E test applications should be run
+        id: matrix-optional
+        run: yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
   job_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We need to add the `always()` check here because the previous step has this as well :(
     # See: https://github.com/actions/runner/issues/2205
-    if: always() && needs.job_e2e_prepare.result == 'success'
+    if: always() && needs.job_e2e_prepare.result == 'success' && needs.job_e2e_prepare.outputs.matrix != '{"include":[]}'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
     timeout-minutes: 15
@@ -870,105 +888,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        is_dependabot:
-          - ${{ github.actor == 'dependabot[bot]' }}
-        test-application:
-          [
-            'angular-17',
-            'angular-18',
-            'astro-4',
-            'aws-lambda-layer-cjs',
-            'aws-serverless-esm',
-            'node-express',
-            'create-react-app',
-            'create-next-app',
-            'create-remix-app',
-            'create-remix-app-legacy',
-            'create-remix-app-v2',
-            'create-remix-app-v2-legacy',
-            'create-remix-app-express',
-            'create-remix-app-express-legacy',
-            'create-remix-app-express-vite-dev',
-            'default-browser',
-            'node-express-esm-loader',
-            'node-express-esm-preload',
-            'node-express-esm-without-loader',
-            'node-express-cjs-preload',
-            'node-otel-sdk-node',
-            'node-otel-custom-sampler',
-            'node-otel-without-tracing',
-            'ember-classic',
-            'ember-embroider',
-            'nextjs-app-dir',
-            'nextjs-13',
-            'nextjs-14',
-            'nextjs-15',
-            'nextjs-turbo',
-            'nextjs-t3',
-            'react-17',
-            'react-19',
-            'react-create-hash-router',
-            'react-router-6-use-routes',
-            'react-router-5',
-            'react-router-6',
-            'solid',
-            'solidstart',
-            'solidstart-spa',
-            'svelte-5',
-            'sveltekit',
-            'sveltekit-2',
-            'sveltekit-2-svelte-5',
-            'sveltekit-2-twp',
-            'tanstack-router',
-            'generic-ts3.8',
-            'node-fastify',
-            'node-fastify-5',
-            'node-hapi',
-            'node-nestjs-basic',
-            'node-nestjs-distributed-tracing',
-            'nestjs-basic',
-            'nestjs-8',
-            'nestjs-distributed-tracing',
-            'nestjs-with-submodules',
-            'nestjs-with-submodules-decorator',
-            'nestjs-basic-with-graphql',
-            'nestjs-graphql',
-            'node-exports-test-app',
-            'node-koa',
-            'node-connect',
-            'nuxt-3',
-            'nuxt-3-min',
-            'nuxt-4',
-            'vue-3',
-            'webpack-4',
-            'webpack-5'
-          ]
-        build-command:
-          - false
-        label:
-          - false
-        # Add any variations of a test app here
-        # You should provide an alternate build-command as well as a matching label
-        include:
-          - test-application: 'create-react-app'
-            build-command: 'test:build-ts3.8'
-            label: 'create-react-app (TS 3.8)'
-          - test-application: 'react-router-6'
-            build-command: 'test:build-ts3.8'
-            label: 'react-router-6 (TS 3.8)'
-          - test-application: 'create-next-app'
-            build-command: 'test:build-13'
-            label: 'create-next-app (next@13)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-13'
-            label: 'nextjs-app-dir (next@13)'
-        exclude:
-          - is_dependabot: true
-            test-application: 'cloudflare-astro'
-          - is_dependabot: true
-            test-application: 'cloudflare-workers'
-
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }}
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
@@ -1069,6 +989,7 @@ jobs:
     # See: https://github.com/actions/runner/issues/2205
     if:
       always() && needs.job_e2e_prepare.result == 'success' &&
+      needs.job_e2e_prepare.outputs.matrix-optional != '{"include":[]}' &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
@@ -1085,58 +1006,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        test-application:
-          [
-            'cloudflare-astro',
-            'cloudflare-workers',
-            'react-send-to-sentry',
-            'node-express-send-to-sentry',
-            'debug-id-sourcemaps',
-          ]
-        build-command:
-          - false
-        assert-command:
-          - false
-        label:
-          - false
-        include:
-          - test-application: 'create-remix-app'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app (sourcemaps)'
-          - test-application: 'create-remix-app-legacy'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app-legacy (sourcemaps)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-canary'
-            label: 'nextjs-app-dir (canary)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-latest'
-            label: 'nextjs-app-dir (latest)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-canary'
-            label: 'nextjs-13 (canary)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-latest'
-            label: 'nextjs-13 (latest)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-canary'
-            label: 'nextjs-14 (canary)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-latest'
-            label: 'nextjs-14 (latest)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-canary'
-            label: 'nextjs-15 (canary)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-latest'
-            label: 'nextjs-15 (latest)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-canary'
-            label: 'nextjs-turbo (canary)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-latest'
-            label: 'nextjs-turbo (latest)'
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }}
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})

dev-packages/e2e-tests/lib/getTestMatrix.ts

@@ -0,0 +1,155 @@
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { dirname } from 'path';
+import { parseArgs } from 'util';
+import { sync as globSync } from 'glob';
+
+interface MatrixInclude {
+  /** The test application (directory) name. */
+  'test-application': string;
+  /** Optional override for the build command to run. */
+  'build-command'?: string;
+  /** Optional override for the assert command to run. */
+  'assert-command'?: string;
+  /** Optional label for the test run. If not set, defaults to value of `test-application`. */
+  label?: string;
+}
+
+interface PackageJsonSentryTestConfig {
+  /** If this is true, the test app is optional. */
+  optional?: boolean;
+  /** Variant configs that should be run in non-optional test runs. */
+  variants?: Partial<MatrixInclude>[];
+  /** Variant configs that should be run in optional test runs. */
+  optionalVariants?: Partial<MatrixInclude>[];
+  /** Skip this test app for matrix generation. */
+  skip?: boolean;
+}
+
+/**
+ * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests.
+ * It checks which test applications are affected by the current changes in the PR and then generates a matrix
+ * including all test apps that have at least one dependency that was changed in the PR.
+ * If no `--base=xxx` is provided, it will output all test applications.
+ *
+ * If `--optional=true` is set, it will generate a matrix of optional test applications only.
+ * Otherwise, these will be skipped.
+ */
+function run(): void {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      base: { type: 'string' },
+      head: { type: 'string' },
+      optional: { type: 'string', default: 'false' },
+    },
+  });
+
+  const { base, head, optional } = values;
+
+  const testApplications = globSync('*/package.json', {
+    cwd: `${__dirname}/../test-applications`,
+  }).map(filePath => dirname(filePath));
+
+  // If `--base=xxx` is defined, we only want to get test applications changed since that base
+  // Else, we take all test applications (e.g. on push)
+  const includedTestApplications = base
+    ? getAffectedTestApplications(testApplications, { base, head })
+    : testApplications;
+
+  const optionalMode = optional === 'true';
+  const includes: MatrixInclude[] = [];
+
+  includedTestApplications.forEach(testApp => {
+    addIncludesForTestApp(testApp, includes, { optionalMode });
+  });
+
+  // We print this to the output, so the GHA can use it for the matrix
+  // eslint-disable-next-line no-console
+  console.log(`matrix=${JSON.stringify({ include: includes })}`);
+}
+
+function addIncludesForTestApp(
+  testApp: string,
+  includes: MatrixInclude[],
+  { optionalMode }: { optionalMode: boolean },
+): void {
+  const packageJson = getPackageJson(testApp);
+
+  const shouldSkip = packageJson.sentryTest?.skip || false;
+  const isOptional = packageJson.sentryTest?.optional || false;
+  const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || [];
+
+  if (shouldSkip) {
+    return;
+  }
+
+  // Add the basic test-application itself, if it is in the current mode
+  if (optionalMode === isOptional) {
+    includes.push({
+      'test-application': testApp,
+    });
+  }
+
+  variants.forEach(variant => {
+    includes.push({
+      'test-application': testApp,
+      ...variant,
+    });
+  });
+}
+
+function getSentryDependencies(appName: string): string[] {
+  const packageJson = getPackageJson(appName) || {};
+
+  const dependencies = {
+    ...packageJson.devDependencies,
+    ...packageJson.dependencies,
+  };
+
+  return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
+}
+
+function getPackageJson(appName: string): {
+  dependencies?: { [key: string]: string };
+  devDependencies?: { [key: string]: string };
+  sentryTest?: PackageJsonSentryTestConfig;
+} {
+  const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
+
+  if (!fs.existsSync(fullPath)) {
+    throw new Error(`Could not find package.json for ${appName}`);
+  }
+
+  return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
+}
+
+run();
+
+function getAffectedTestApplications(
+  testApplications: string[],
+  { base = 'develop', head }: { base?: string; head?: string },
+): string[] {
+  const additionalArgs = [`--base=${base}`];
+
+  if (head) {
+    additionalArgs.push(`--head=${head}`);
+  }
+
+  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
+    .toString()
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+
+  // If something in e2e tests themselves are changed, just run everything
+  if (affectedProjects.includes('@sentry-internal/e2e-tests')) {
+    return testApplications;
+  }
+
+  return testApplications.filter(testApp => {
+    const sentryDependencies = getSentryDependencies(testApp);
+    return sentryDependencies.some(dep => affectedProjects.includes(dep));
+  });
+}

dev-packages/e2e-tests/package.json

@@ -14,11 +14,13 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp node_modules pnpm-lock.yaml && yarn clean:test-applications",
+    "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts",
+    "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.sveltekit,pnpm-lock.yaml} .last-run.json && pnpm store prune"
   },
   "devDependencies": {
     "@types/glob": "8.0.0",
-    "@types/node": "^14.18.0",
+    "@types/node": "^18.0.0",
     "dotenv": "16.0.3",
     "esbuild": "0.20.0",
     "glob": "8.0.3",

dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json

@@ -26,5 +26,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }