WIP Mi/pw exp

.github/workflows/positron-merge-to-main.yml

@@ -27,10 +27,56 @@ permissions:
   contents: read
 
 jobs:
-  linux:
-    name: Tests on Linux
+  e2e-playwright:
+    timeout-minutes: 20
     runs-on: ubuntu-latest-8x
-    timeout-minutes: 45
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      POSITRON_BUILD_NUMBER: 0 # CI skips building releases
+      _R_CHECK_FUTURE_FILE_TIMESTAMPS_: false # this check can be flaky in the R pkg tests
+      _R_CHECK_CRAN_INCOMING_: false
+      _R_CHECK_SYSTEM_CLOCK_: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Cache node_modules, build, extensions, and remote
+        uses: ./.github/actions/cache-multi-paths
+
+      - name: Setup Build and Compile
+        uses: ./.github/actions/setup-build-env
+
+      - name: Setup E2E Test Environment
+        uses: ./.github/actions/setup-test-env
+        with:
+          aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
+          aws-region: ${{ secrets.QA_AWS_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compile E2E Tests
+        run: yarn --cwd test/automation compile && yarn --cwd test/smoke compile
+
+      - name: Run Playwright Tests (Electron)
+        env:
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
+        id: e2e-playwright-tests
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --grep @pr --workers 2
+
+      - name: Upload Playwright Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 7
+
+  e2e-mocha:
+    runs-on: ubuntu-latest-8x
+    timeout-minutes: 30
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -63,17 +109,6 @@ jobs:
           aws-region: ${{ secrets.QA_AWS_REGION }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Compile Integration Tests
-        run: yarn --cwd test/integration/browser compile
-
-      - name: Run Unit Tests (node.js)
-        id: nodejs-unit-tests
-        run: yarn test-node
-
-      - name: Run Integration Tests (Electron)
-        id: electron-integration-tests
-        run: DISPLAY=:10 ./scripts/test-integration-pr.sh
-
       - name: Run Smoke Tests (Electron)
         if: success() || failure()
         env:
@@ -110,11 +145,59 @@ jobs:
     outputs:
       target: ${{ env.SMOKETEST_TARGET }}
 
+  unit-integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      POSITRON_BUILD_NUMBER: 0 # CI skips building releases
+      _R_CHECK_FUTURE_FILE_TIMESTAMPS_: false # this check can be flaky in the R pkg tests
+      _R_CHECK_CRAN_INCOMING_: false
+      _R_CHECK_SYSTEM_CLOCK_: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Cache node_modules, build, extensions, and remote
+        uses: ./.github/actions/cache-multi-paths
+
+      - name: Setup Build and Compile
+        uses: ./.github/actions/setup-build-env
+
+      - name: Install Positron License
+        uses: ./.github/actions/install-license
+        with:
+          github-token: ${{ secrets.POSITRON_GITHUB_PAT }}
+          license-key: ${{ secrets.POSITRON_DEV_LICENSE }}
+
+      # one unit test needs this: Can list tables and fields from R connections
+      - name: Setup R
+        uses: ./.github/actions/install-r
+        with:
+          version: "4.4.0"
+
+      - name: Compile Integration Tests
+        run: yarn --cwd test/integration/browser compile
+
+      - name: Run Unit Tests (node.js)
+        id: nodejs-unit-tests
+        run: yarn test-node
+
+      - name: Run Integration Tests (Electron)
+        id: electron-integration-tests
+        run: DISPLAY=:10 ./scripts/test-integration-pr.sh
+
+    outputs:
+      target: ${{ env.SMOKETEST_TARGET }}
+
   slack-notification:
     name: "Send Slack notification"
     runs-on: ubuntu-latest
-    needs: linux
-    if: ${{ failure() && needs.linux.outputs.target  == 'smoketest-all' }}
+    needs: [unit-integration, e2e-mocha, e2e-playwright]
+    if: ${{ failure() && needs.unit-integration.outputs.target  == 'smoketest-all' }}
     steps:
       - name: "Send Slack notification"
         uses: testlabauto/action-test-results-to-slack@v0.0.6

.github/workflows/positron-pull-requests.yml

@@ -6,7 +6,7 @@ on:
       - main
 
 jobs:
-  positron-ci:
+  tests:
     uses: ./.github/workflows/positron-merge-to-main.yml
     secrets: inherit
     with:

.gitignore

@@ -25,5 +25,7 @@ product.overrides.json
 
 # --- Start Positron ---
 .Rproj.user
+/playwright-report/
+/blob-report/
+/playwright/.cache/
 # --- End Positron ---
-

package.json

@@ -130,7 +130,7 @@
     "yazl": "^2.4.3"
   },
   "devDependencies": {
-    "@playwright/test": "^1.46.1",
+    "@playwright/test": "^1.48.0",
     "@swc/core": "1.3.62",
     "@types/cookie": "^0.3.3",
     "@types/debug": "^4.1.5",

playwright.config.ts

@@ -0,0 +1,70 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2024 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { defineConfig } from '@playwright/test';
+import { CustomTestOptions } from './test/smoke/src/e2e/_test.setup';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig<CustomTestOptions>({
+	globalSetup: require.resolve('./test/smoke/src/e2e/_global.setup.ts'),
+	testDir: './test/smoke/src/e2e',
+	testMatch: '*.test.ts',
+	fullyParallel: false, // Run individual tests in parallel
+	forbidOnly: !!process.env.CI,
+	retries: process.env.CI ? 1 : 0,
+	workers: 3, // Number of parallel workers (tests will run in parallel)
+	timeout: 2 * 60 * 1000, // test timeout is 2 minutes
+	reportSlowTests: {
+		max: 10,
+		threshold: 60 * 1000, // 1 minute
+	},
+	reporter: process.env.CI
+		? [
+			['github'],
+			['junit', { outputFile: 'test-results/junit.xml' }],
+			['html']
+		]
+		: [
+			['list'],
+			['html', { open: 'on-failure' }]
+		],
+
+
+	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+	use: {
+		headless: false,
+		trace: 'off', // we are manually handling tracing in _test.setup.ts
+	},
+
+	projects: [
+		{
+			name: 'e2e-electron',
+			use: {
+				web: false,
+				artifactDir: 'e2e-electron'
+			},
+		},
+		{
+			name: 'e2e-chromium',
+			use: {
+				web: true,
+				artifactDir: 'e2e-chromium',
+				headless: false,
+			},
+			grep: /@web/
+		},
+	],
+});

test/automation/src/application.ts

@@ -104,9 +104,11 @@ export class Application {
 		await this._code?.startTracing(name);
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
-		await this._code?.stopTracing(name, persist);
+	// --- Start Positron ---
+	async stopTracing(name: string, persist: boolean, customPath?: string): Promise<void> {
+		await this._code?.stopTracing(name, persist, customPath);
 	}
+	// --- End Positron ---
 
 	private async startApplication(extraArgs: string[] = []): Promise<Code> {
 		const code = this._code = await launch({

test/automation/src/code.ts

@@ -121,9 +121,11 @@ export class Code {
 		return await this.driver.startTracing(name);
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
-		return await this.driver.stopTracing(name, persist);
+	// --- Start Positron ---
+	async stopTracing(name: string, persist: boolean, customPath?: string): Promise<void> {
+		return await this.driver.stopTracing(name, persist, customPath);
 	}
+	// --- End Positron ---
 
 	async dispatchKeybinding(keybinding: string): Promise<void> {
 		await this.driver.dispatchKeybinding(keybinding);

test/automation/src/playwrightBrowser.ts

@@ -105,7 +105,9 @@ async function launchBrowser(options: LaunchOptions, endpoint: string) {
 
 	if (tracing) {
 		try {
-			await measureAndLog(() => context.tracing.start({ screenshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			// --- Start Positron ---
+			await measureAndLog(() => context.tracing.start({ screenshots: true, snapshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			// --- End Positron ---
 		} catch (error) {
 			logger.log(`Playwright (Browser): Failed to start playwright tracing (${error})`); // do not fail the build when this fails
 		}

test/automation/src/playwrightDriver.ts

@@ -57,7 +57,9 @@ export class PlaywrightDriver {
 		}
 	}
 
-	async stopTracing(name: string, persist: boolean): Promise<void> {
+	// --- Start Positron ---
+	async stopTracing(name: string, persist: boolean = true, customPath?: string): Promise<void> {
+		// --- End Positron ---
 		if (!this.options.tracing) {
 			return; // tracing disabled
 		}
@@ -67,18 +69,18 @@ export class PlaywrightDriver {
 			if (persist) {
 				// --- Start Positron ---
 				// Positron: Windows has issues with long paths, shortened the name
-				persistPath = join(this.options.logsPath, `trace-${PlaywrightDriver.traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
+				persistPath = customPath || join(this.options.logsPath, `trace-${PlaywrightDriver.traceCounter++}-${name.replace(/\s+/g, '-')}.zip`);
 				// --- End Positron ---
 			}
-
 			await measureAndLog(() => this.context.tracing.stopChunk({ path: persistPath }), `stopTracing for ${name}`, this.options.logger);
 
 			// To ensure we have a screenshot at the end where
 			// it failed, also trigger one explicitly. Tracing
 			// does not guarantee to give us a screenshot unless
 			// some driver action ran before.
-			if (persist) {
-				// --- Start Positron ---
+			// --- Start Positron ---
+			if (!customPath && persist) { // only take screenshots if running via mocha.
+
 				await this.takeScreenshot(`${name}`);
 				// --- End Positron ---
 			}

test/automation/src/playwrightElectron.ts

@@ -45,7 +45,9 @@ async function launchElectron(configuration: IElectronConfiguration, options: La
 
 	if (tracing) {
 		try {
-			await measureAndLog(() => context.tracing.start({ screenshots: true, /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			// --- Start Positron ---
+			await measureAndLog(() => context.tracing.start({ screenshots: true, snapshots: true /* remaining options are off for perf reasons */ }), 'context.tracing.start()', logger);
+			// --- End Positron ---
 		} catch (error) {
 			logger.log(`Playwright (Electron): Failed to start playwright tracing (${error})`); // do not fail the build when this fails
 		}

test/smoke/README.md

@@ -167,7 +167,7 @@ Make sure that you have followed the [Machine Setup](https://connect.posit.it/po
 
 ### Test Dependencies
 
-Several tests use [QA Content Examples](https://github.com/posit-dev/qa-example-content). You will need to install the dependencies for those projects.  A few current tests also use additional packages. You can look in the [positron-full-test.yml](https://github.com/posit-dev/positron/blob/39a01b71064e2ef3ef5822c95691a034b7e0194f/.github/workflows/positron-full-test.yml) Github action for the full list.
+Several tests use [QA Content Examples](https://github.com/posit-dev/qa-example-content). You will need to install the dependencies for those projects. A few current tests also use additional packages. You can look in the [positron-full-test.yml](https://github.com/posit-dev/positron/blob/39a01b71064e2ef3ef5822c95691a034b7e0194f/.github/workflows/positron-full-test.yml) Github action for the full list.
 
 ## Running Tests
 
@@ -235,9 +235,9 @@ yarn smoketest-pr --build /Applications/Positron.app --parallel --jobs 3
 
 ## Test Project
 
-Before any of the tests start executing the test framework clones down the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.  This repo contains R and Python files that are run by the automated tests and also includes data files (such as Excel, SQLite, & parquet) that support the test scripts.  If you make additions to QA Content Examples for a test, please be sure that the data files are free to use in a public repository.
+Before any of the tests start executing the test framework clones down the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo. This repo contains R and Python files that are run by the automated tests and also includes data files (such as Excel, SQLite, & parquet) that support the test scripts. If you make additions to QA Content Examples for a test, please be sure that the data files are free to use in a public repository.
 
-For Python, add any package requirements to the `requirements.txt` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.  We generally do NOT pin them to a specific version, as test can be run against different versions of python and conflicts could arise.  If this becomes a problem, we can revisit this mechanism.
+For Python, add any package requirements to the `requirements.txt` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo. We generally do NOT pin them to a specific version, as test can be run against different versions of python and conflicts could arise. If this becomes a problem, we can revisit this mechanism.
 
 For R, add any package requirements to the "imports" section of the `DESCRIPTION` file in the root of the [QA Content Examples](https://github.com/posit-dev/qa-example-content) repo.
 
@@ -262,26 +262,26 @@ await app.workbench.quickaccess.runCommand('workbench.action.toggleDevTools');`
 
 ### Playwright Traces
 
-Note that in launch.json for `Launch Smoke Test` we are passing the `--tracing` argument for you.  This will result in Playwright traces being generated locally for you when tests fail at `.build/logs/smoke-tests-electron/{testCase}`.  Note that for command line runs you will need to pass this arg yourself to get the trace file(s).
+Note that in launch.json for `Launch Smoke Test` we are passing the `--tracing` argument for you. This will result in Playwright traces being generated locally for you when tests fail at `.build/logs/smoke-tests-electron/{testCase}`. Note that for command line runs you will need to pass this arg yourself to get the trace file(s).
 
 ## Running Tests in Github Actions
 
-New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](https://github.com/posit-dev/positron/actions/workflows/positron-full-test.yml).  In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system.  Also, you can easily run your new tests against a branch to verify them before merge.  Simply pick the branch after you click on "Run Workflow".  Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly.  To do this, skip the runs of the unit and integration tests.
+New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](https://github.com/posit-dev/positron/actions/workflows/positron-full-test.yml). In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system. Also, you can easily run your new tests against a branch to verify them before merge. Simply pick the branch after you click on "Run Workflow". Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly. To do this, skip the runs of the unit and integration tests.
 
 ### Github Actions Test Artifacts
 
-When a run is complete, you can debug any test failures that occurred using the uploaded run artifacts.  The artifacts are available as a ZIP file from inside the workflow run.  Each artifact zip contains: a folder for each test file and an overall run log.  Inside the folder corresponding to each test file, you will find zip files that are Playwright traces.  Note that the trace files are only present for failed cases.
+When a run is complete, you can debug any test failures that occurred using the uploaded run artifacts. The artifacts are available as a ZIP file from inside the workflow run. Each artifact zip contains: a folder for each test file and an overall run log. Inside the folder corresponding to each test file, you will find zip files that are Playwright traces. Note that the trace files are only present for failed cases.
 
-Playwright traces can be drag and dropped to the [Trace Viewer](https://trace.playwright.dev/).  The trace will usually give you a good visualization of the failed test, but they can be sparse on details.  More details are available from the run log (smoke-test-runner.log).  It has a start and end marker for each test case.
+Playwright traces can be drag and dropped to the [Trace Viewer](https://trace.playwright.dev/). The trace will usually give you a good visualization of the failed test, but they can be sparse on details. More details are available from the run log (e2e-test-runner.log). It has a start and end marker for each test case.
 
 ## Notes About Updating Specific Tests
 
 ### Plot Tests That Use Resemblejs
 
-In order to get the "golden screenshots" used for plot comparison is CI, you will need to temporarily uncomment the line of code marked with `capture master image in CI` or add a similar line of code for a new case.  We must use CI taken snapshots because if the "golden screenshots" are taken locally, they will differ too much from the CI images to be useable with a proper threshold.  You can't compare the current runtime plot against a snapshot until you have established a baseline screenshot from CI that is saved to `test/smoke/plots`.
+In order to get the "golden screenshots" used for plot comparison is CI, you will need to temporarily uncomment the line of code marked with `capture master image in CI` or add a similar line of code for a new case. We must use CI taken snapshots because if the "golden screenshots" are taken locally, they will differ too much from the CI images to be useable with a proper threshold. You can't compare the current runtime plot against a snapshot until you have established a baseline screenshot from CI that is saved to `test/smoke/plots`.
 
 ## Tests run on PRs
 
-If you think your test should be run when PRs are created, add the string `#pr` to its name.  The existing #pr cases were selected to give good overall coverage while keeping the overall execution time down to ten minutes or less.  If your new test functionality covers a part of the application that no other tests cover, it is probably a good idea to include it in the #pr set.
+If you think your test should be run when PRs are created, add the string `#pr` to its name. The existing #pr cases were selected to give good overall coverage while keeping the overall execution time down to ten minutes or less. If your new test functionality covers a part of the application that no other tests cover, it is probably a good idea to include it in the #pr set.
 
 <!-- End Positron -->

test/smoke/package.json

@@ -26,6 +26,7 @@
     "@types/node-fetch": "^2.5.10",
     "@types/resemblejs": "4.1.3",
     "@types/rimraf": "3.0.2",
+    "archiver": "^7.0.1",
     "npm-run-all": "^4.1.5",
     "watch": "^1.0.2"
   }

test/smoke/src/areas/positron/console/consoleInput.test.ts

@@ -118,7 +118,7 @@ cat(sprintf('Hello %s!\n', val))`;
 
 			await expect(async () => {
 				await app.workbench.positronConsole.typeToConsole(inputCode);
-			}).toPass({ timeout: 600 });
+			}).toPass({ timeout: 60000 });
 
 			const activeConsole = app.workbench.positronConsole.activeConsole;