test: add requirement(s) in the form of executable BDD scenarios (#557)

* test: write e2e testcases as bdd scenarios

* test: enable running tests from the test explorer again

* chore: enable Gherkin linter

* test: re-structure into pages and steps

* chore: fix linter findings

* chore: update tests

* chore: fix tests

* Fix scenario description being seen as Given steps

* chore: fix linter findings

.devcontainer/cpp/devcontainer.json

@@ -20,10 +20,16 @@
   "customizations": {
     "vscode": {
       "settings": {
+        "cucumberautocomplete.steps": [".devcontainer/cpp/e2e/features/steps/*.steps.ts"],
+        "cucumberautocomplete.strictGherkinCompletion": false,
+        "cucumberautocomplete.strictGherkinValidation": false,
+        "cucumberautocomplete.smartSnippets": true,
+        "cucumberautocomplete.onTypeFormat": true,
         "files.insertFinalNewline": true,
         "files.trimTrailingWhitespace": true
       },
       "extensions": [
+        "alexkrechik.cucumberautocomplete@3.0.5",
         "github.copilot@1.208.0",
         "github.vscode-github-actions@0.26.3",
         "github.vscode-pull-request-github@0.90.0",

.devcontainer/cpp/e2e/features/compilation.feature

@@ -0,0 +1,18 @@
+Feature: Compile source code into working software
+
+  As a developer
+  In order to generate working software
+  Source code needs to be compiled successfully
+
+  Scenario: Compile valid source code into working software targeting the host architecture
+
+    Compiling valid source code into working software, able to run on the host architecture,
+    can be necessary in several scenarios; for example when:
+
+    - the host is the deployment target
+    - running tests on the host
+    - building plug-ins, extensions, code generators, or other additional tools that need to run on the host
+
+    Given the default build configuration is selected
+    When the configuration "host" is built
+    Then the output should contain "Build finished with exit code 0"

.devcontainer/cpp/e2e/features/pages/authentication.pom.ts

@@ -0,0 +1,37 @@
+import { type Page } from '@playwright/test';
+import * as OTPAuth from 'otpauth';
+import { STORAGE_STATE } from '../../playwright.config';
+
+export class AuthenticationPage {
+  readonly page: Page;
+
+  constructor(page: Page) {
+    this.page = page;
+  }
+
+  async authenticate() {
+    await this.page.goto('https://github.com/login');
+    await this.page.getByLabel('Username or email address').fill(process.env.GITHUB_USER!);
+    await this.page.getByLabel('Password').fill(process.env.GITHUB_PASSWORD!);
+    await this.page.getByRole('button', { name: 'Sign in', exact: true }).click();
+
+    let totp = new OTPAuth.TOTP({
+      issuer: 'GitHub',
+      label: 'GitHub',
+      algorithm: 'SHA1',
+      digits: 6,
+      period: 30,
+      secret: process.env.GITHUB_TOTP_SECRET!
+    });
+
+    let code = totp.generate();
+    await this.page.getByPlaceholder('XXXXXX').fill(code);
+
+    // Wait until the page receives the cookies.
+    //
+    // Sometimes login flow sets cookies in the process of several redirects.
+    // Wait for the final URL to ensure that the cookies are actually set.
+    await this.page.waitForURL('https://github.com/');
+    await this.page.context().storageState({ path: STORAGE_STATE });
+  }
+}

.devcontainer/cpp/e2e/tests/codespace.pom.ts → .devcontainer/cpp/e2e/features/pages/codespace.pom.ts

@@ -100,4 +100,8 @@ export class CodespacePage {
     await this.page.getByRole('treeitem', { name: name }).locator('a').click();
     await expect(this.page.locator('[id="workbench.parts.editor"]')).toContainText(name);
   }
+
+  async buildSelectedTarget() {
+    await this.page.getByRole('button', { name: 'Build the selected target' }).click();
+  }
 }

.devcontainer/cpp/e2e/features/steps/compilation.steps.ts

@@ -0,0 +1,15 @@
+import { expect } from "@playwright/test";
+import { Given, When, Then } from "./fixtures";
+
+Given("the default build configuration is selected", async () => {
+  // No-op
+});
+
+When("the configuration {string} is built", async ({ codespacePage }, configuration: string) => {
+  await codespacePage.page.getByRole('button', { name: 'Build the selected target' }).click();
+  await codespacePage.page.getByLabel(configuration).locator('a').click();
+});
+
+Then("the output should contain {string}", async ({ codespacePage }, expectedOutput: string) => {
+  await expect(codespacePage.outputPanel).toContainText(expectedOutput, { timeout: 5 * 60 * 1000 });
+});

.devcontainer/cpp/e2e/features/steps/fixtures.ts

@@ -0,0 +1,24 @@
+import { AuthenticationPage } from '../pages/authentication.pom';
+import { CodespacePage } from '../pages/codespace.pom';
+import { test as base, createBdd } from 'playwright-bdd';
+
+export const test = base.extend<{ codespacePage: CodespacePage }, { authenticationPage: AuthenticationPage }>({
+    authenticationPage: [async ({ browser }, use) => {
+        let authenticationPage = new AuthenticationPage(await browser.newPage());
+        await authenticationPage.authenticate();
+
+        await use(authenticationPage);
+      }, { scope: 'worker', auto: true }
+    ],
+    codespacePage: async ({ page }, use) => {
+        const codespacePage = new CodespacePage(page);
+        await codespacePage.goto();
+        await codespacePage.areExtensionsActive(['Testing', 'SonarLint', 'CMake', 'Live Share', 'GitHub Pull Requests']);
+
+        await use(codespacePage);
+
+        await codespacePage.executeInTerminal('git clean -fdx');
+    },
+});
+
+export const { Given, When, Then } = createBdd(test);

.devcontainer/cpp/e2e/playwright.config.ts

@@ -1,12 +1,18 @@
 import { defineConfig, devices } from '@playwright/test';
+import { defineBddConfig } from "playwright-bdd";
 import path from 'path';
 
-require('dotenv').config();
+require('dotenv').config({ path: path.join(__dirname, '..', '..', '..', '.env') });
 
 export const STORAGE_STATE = path.join(__dirname, 'playwright/.auth/user.json');
 
+const testDir = defineBddConfig({
+  features: "features/*.feature",
+  steps: ["features/steps/*.ts"],
+});
+
 export default defineConfig({
-  testDir: './tests',
+  testDir,
   fullyParallel: false,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
@@ -16,14 +22,12 @@ export default defineConfig({
     trace: 'on-first-retry'
   },
   projects: [
-    { name: 'setup', testMatch: '**/*.setup.ts' },
     {
       name: 'chromium',
       use: {
         ...devices['Desktop Chrome'],
         storageState: STORAGE_STATE
-      },
-      dependencies: ['setup']
+      }
     }
   ]
 });

.devcontainer/cpp/e2e/workspace/CMakePresets.json

@@ -12,8 +12,6 @@
     },
     {
       "name": "host",
-      "displayName": "host",
-      "description": "Build for host",
       "inherits": "defaults"
     }
   ],

.github/linters/.gherkin-lintrc

@@ -0,0 +1,22 @@
+{
+  "file-name": ["on", {"style": "kebab-case"}],
+  "new-line-at-eof": ["on", "yes"],
+  "no-background-only-scenario": "off",
+  "no-dupe-feature-names": "on",
+  "no-dupe-scenario-names": ["on", "in-feature"],
+  "no-duplicate-tags": "on",
+  "no-empty-background": "on",
+  "no-empty-file": "on",
+  "no-files-without-scenarios": "on",
+  "no-multiple-empty-lines": "on",
+  "no-partially-commented-tag-lines": "on",
+  "no-scenario-outlines-without-examples": "on",
+  "no-superfluous-tags": "on",
+  "no-trailing-spaces": "on",
+  "no-unnamed-features": "on",
+  "no-unnamed-scenarios": "on",
+  "no-unused-variables": "on",
+  "one-space-between-tags": "on",
+  "scenario-size": ["on", { "steps-length": {"Background": 5}}],
+  "use-and": "on"
+}

.github/workflows/linting-formatting.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: oxsecurity/megalinter/flavors/documentation@c217fe8f7bc9207062a084e989bd97efd56e7b9a # v8.0.0
+      - uses: oxsecurity/megalinter@c217fe8f7bc9207062a084e989bd97efd56e7b9a # v8.0.0
         env:
           APPLY_FIXES: all
           VALIDATE_ALL_CODEBASE: true

.gitignore

@@ -1,3 +1,4 @@
+.features-gen/
 .xwin-cache/
 .xwin-hash/
 build/

.mega-linter.yml

@@ -1,6 +1,7 @@
 ENABLE:
   - ACTION
   - DOCKERFILE
+  - GHERKIN
   - JSON
   - MARKDOWN
   - REPOSITORY
@@ -9,14 +10,15 @@ ENABLE:
 DISABLE_LINTERS:
   - MARKDOWN_MARKDOWN_LINK_CHECK
   - REPOSITORY_DEVSKIM
+  - REPOSITORY_DUSTILOCK
   - REPOSITORY_KICS
   - REPOSITORY_SEMGREP
   - JSON_JSONLINT
   - SPELL_CSPELL
 SARIF_REPORTER: true
 PRINT_ALPACA: false
 SHOW_SKIPPED_LINTERS: false
-FILTER_REGEX_EXCLUDE: (CHANGELOG.md)
+FILTER_REGEX_EXCLUDE: (CHANGELOG.md|package-lock.json)
 # tasks.json is wrongfully matched against another schema,
 # and schemas for .vscode/[tasks.json|launch.json] are built
 # dynamically based upon context (e.g. installed extensions)

README.md

@@ -113,7 +113,7 @@ A test task is available to run the included `bats` tests. Choose `Tasks: Run Te
 
 #### Running the Acceptance Tests
 
-Create an .env file with the following contents, this assumes a GitHub account that has rights to create a Codespace on this repository and is configured for time-based one-time password (TOTP) two-factor authentication (2FA).
+Create a .env file in the root of the workspace with the following contents, this assumes a GitHub account that has rights to create a Codespace on this repository and is configured for time-based one-time password (TOTP) two-factor authentication (2FA).
 
 ```dotenv
 GITHUB_USER=