Add e2e tests

.github/workflows/README.md

@@ -11,8 +11,8 @@ Each app should have:
 - `ci-[app_name]`: must be created; should run linting and testing
 - `ci-[app_name]-vulnerability-scans`: calls `vulnerability-scans`
   - Based on [ci-app-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-vulnerability-scans.yml)
-- `ci-[app_name]-pr-environment-update.yml`: calls `pr-environment-update.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md))
-  - Based on [ci-app-pr-environment-update.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-update.yml)
+- `ci-[app_name]-pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md))
+  - Based on [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml)
 - `ci-[app_name]-pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md))
   - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml)
 

.github/workflows/ci-app-pr-environment-update.yml → .github/workflows/ci-app-pr-environment-checks.yml

@@ -1,4 +1,4 @@
-name: CI App PR Environment Update
+name: CI App PR Environment Checks
 on:
   workflow_dispatch:
     inputs:
@@ -13,7 +13,7 @@ on:
 jobs:
   update:
     name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise
-    uses: ./.github/workflows/pr-environment-update.yml
+    uses: ./.github/workflows/pr-environment-checks.yml
     with:
       app_name: "app"
       environment: "dev"

.github/workflows/e2e-tests.yml

@@ -0,0 +1,37 @@
+name: E2E Tests
+
+on:
+  workflow_call:
+    inputs:
+      service_endpoint:
+        required: true
+        type: string
+      app_name:
+        required: false
+        type: string
+
+jobs:
+  e2e:
+    name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install Playwright browsers
+        run: make e2e-setup-ci
+
+      - name: Run e2e tests
+        run: make e2e-test APP_NAME=${{ inputs.app_name }} BASE_URL=${{ inputs.service_endpoint }}
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: ./e2e/playwright-report

.github/workflows/pr-environment-update.yml → .github/workflows/pr-environment-checks.yml

@@ -36,6 +36,9 @@ jobs:
 
     concurrency: pr-environment-${{ inputs.pr_number }}
 
+    outputs:
+      service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }}
+
     steps:
       - uses: actions/checkout@v4
 
@@ -52,6 +55,19 @@ jobs:
           environment: ${{ inputs.environment }}
 
       - name: Update environment
-        run: ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}"
+        id: update-environment
+        run: |
+            ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}"
+            service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint)
+            echo "service_endpoint=${service_endpoint}"
+            echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT"
         env:
           GH_TOKEN: ${{ github.token }}
+
+  e2e-tests:
+    name: Run E2E Tests
+    needs: [update]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      service_endpoint: ${{ needs.update.outputs.service_endpoint }}
+      app_name: ${{ inputs.app_name }}

Makefile

@@ -59,9 +59,9 @@ __check_defined = \
 	release-image-name \
 	release-image-tag \
 	release-publish \
-	release-run-database-migrations
-
-
+	release-run-database-migrations \
+	e2e-setup \
+	e2e-test
 
 infra-set-up-account: ## Configure and create resources for current AWS profile and save tfbackend file to infra/accounts/$ACCOUNT_NAME.ACCOUNT_ID.s3.tfbackend
 	@:$(call check_defined, ACCOUNT_NAME, human readable name for account e.g. "prod" or the AWS account alias)
@@ -222,6 +222,28 @@ release-image-name: ## Prints the image name of the release image
 release-image-tag: ## Prints the image tag of the release image
 	@echo $(IMAGE_TAG)
 
+##############################
+## End-to-end (E2E) Testing ##
+##############################
+
+e2e-setup: ## Setup end-to-end tests
+	@cd e2e && npm install
+	@cd e2e && npx playwright install --with-deps
+
+e2e-setup-ci: ## Install system dependencies, Node dependencies, and Playwright browsers
+	sudo apt-get update
+	sudo apt-get install -y libwoff1 libopus0 libvpx7 libevent-2.1-7 libopus0 libgstreamer1.0-0 \
+	libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0 libharfbuzz-icu0 libhyphen0 \
+	libenchant-2-2 libflite1 libgles2 libx264-dev
+	cd e2e && npm ci
+	cd e2e && npx playwright install --with-deps
+
+
+e2e-test: ## Run end-to-end tests
+	@:$(call check_defined, APP_NAME, You must pass in a specific APP_NAME)
+	@:$(call check_defined, BASE_URL, You must pass in a BASE_URL)
+	@cd e2e/$(APP_NAME) && APP_NAME=$(APP_NAME) BASE_URL=$(BASE_URL) npx playwright test $(E2E_ARGS)
+
 ########################
 ## Scripts and Helper ##
 ########################

docs/e2e/e2e-checks.md

@@ -0,0 +1,73 @@
+# End-to-End (E2E) Tests
+
+## Overview
+
+This repository uses [Playwright](https://playwright.dev/) to perform end-to-end (E2E) tests. The tests can be run locally, but also run on [Pull Request preview environments](../infra/pull-request-environments.md). This ensures that any new code changes are validated through E2E tests before being merged.
+
+## Folder Structure
+In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below:
+```
+- e2e
+  - playwright.config.js
+  - app/
+    - playwright.config.js
+    - tests/
+      - index.spec.js
+  - app2/
+    - playwright.config.js
+    - tests/
+      - index.spec.js
+```
+
+Some highlights:
+- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config.
+- Snapshots will be output locally or in the artifacts of the CI job
+- HTML reports are output to the `playwright-report` folder
+- Parallelism limited on CI to ensure stable execution
+- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing)
+
+
+## Running Locally
+
+### Running Locally From the Root Directory
+
+Make targets are setup to easily pass in a particular app name and URL to run tests against
+
+```bash
+make e2e-setup # install playwright deps
+make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 # run tests on a particular app
+```
+
+### Running Locally From the `./e2e` Directory
+
+If you prefer to run package.json run scripts, you can do so from the e2e folder:
+
+```
+cd e2e
+
+npm install
+
+APP_NAME=app npm run e2e-test
+```
+
+### PR Environments
+
+The E2E tests are triggered in PR preview environments on each PR update. For more information on how PR environments work, please refer to [PR Environments Documentation](../infra/pull-request-environments.md).
+
+### Workflows
+
+The following workflows trigger E2E tests:
+- [PR Environment Update](../../.github/workflows/pr-environment-checks.yml)
+- [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml)
+
+The [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) takes a `service_endpoint` URL and an `app_name` to run the tests against specific configurations for your app.
+
+## Configuration
+
+The E2E tests are configured using the following files:
+- [Base Configuration](../../e2e/playwright.config.js)
+- [App-specific Configuration](../../e2e/app/playwright.config.js)
+
+The app-specific configuration files extend the common base configuration.
+
+By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e/app/playwright.config.js`).

docs/infra/pull-request-environments.md

@@ -23,12 +23,12 @@ This guidance is not strict. It is still okay to combine database migrations and
 
 Pull request environments are created by GitHub Actions workflows. There are two reusable callable workflows that manage pull request environments:
 
-- [pr-environment-update.yml](/.github/workflows/pr-environment-update.yml) - creates or updates a temporary environment in a separate Terraform workspace for a given application and pull request
+- [pr-environment-checks.yml](/.github/workflows/pr-environment-checks.yml) - creates or updates a temporary environment in a separate Terraform workspace for a given application and pull request
 - [pr-environment-destroy.yml](/.github/workflows/pr-environment-destroy.yml) - destroys a temporary environment and workspace for a given application and pull request
 
 Using these reusable workflows, configure PR environments for each application with application-specific workflows:
 
-- `ci-[app_name]-pr-environment-update.yml`
-  - Based on [ci-app-pr-environment-update.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-update.yml)
+- `ci-[app_name]-pr-environment-checks.yml`
+  - Based on [ci-app-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-checks.yml)
 - `ci-[app_name]-pr-environment-destroy.yml`
   - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml)

e2e/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+*.png*

e2e/app/playwright.config.js

@@ -0,0 +1,12 @@
+import baseConfig from '../playwright.config';
+import { deepMerge } from '../util';
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig(deepMerge(
+    baseConfig,
+    {
+      use: {
+        baseUrl: baseConfig.use.baseUrl || "localhost:3000"
+      },
+    }
+  ));

e2e/app/tests/index.spec.js

@@ -0,0 +1,31 @@
+const { test, expect } = require('@playwright/test');
+
+import AxeBuilder from '@axe-core/playwright';
+
+test.describe('Generic Webpage Tests', () => {
+  test('should load the webpage successfully', async ({ page }) => {
+    const response = await page.goto('/');
+    const title = await page.title();
+    await expect(response.status()).toBe(200);
+  });
+
+  test('should take a screenshot of the webpage', async ({ page }) => {
+    await page.goto('/');
+    await page.screenshot({ path: 'example-screenshot.png', fullPage: true });
+  });
+
+  // https://playwright.dev/docs/accessibility-testing
+  test('should not have any automatically detectable accessibility issues', async ({ page }) => {
+    await page.goto('/');
+    const accessibilityScanResults = await new AxeBuilder({ page }).analyze();
+    expect(accessibilityScanResults.violations).toEqual([]);
+  });
+
+  // Example test of finding a an html element on the index/home page
+  // test('should check for an element to be visible', async ({ page }) => {
+  //   await page.goto('/');
+  //   const element = page.locator('h1');
+  //   await expect(element).toBeVisible();
+  // });
+
+});