Backport of [ui, tests] Various acceptance test fixups (v main) into release/1.9.x

.github/workflows/test-ui.yml

@@ -2,14 +2,14 @@ name: test-ui
 on:
   pull_request:
     paths:
-      - 'ui/**'
+      - "ui/**"
   push:
     branches:
       - main
       - release/**
       - test-ui
     paths:
-      - 'ui/**'
+      - "ui/**"
 
 jobs:
   pre-test:
@@ -36,14 +36,16 @@ jobs:
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
-    continue-on-error: true
     defaults:
       run:
         working-directory: ui
     strategy:
+      fail-fast: false
       matrix:
         partition: [1, 2, 3, 4]
         split: [4]
+        # Note: If we ever change the number of partitions, we'll need to update the
+        # finalize.combine step to match
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-js
@@ -60,11 +62,33 @@ jobs:
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: ember exam
+        id: ember_exam
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
-        run: yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }}
-
+        run: |
+          yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
+        continue-on-error: true
+      - name: Express failure
+        if: steps.ember_exam.outcome == 'failure'
+        run: |
+          echo "Tests failed in ember-exam for partition ${{ matrix.partition }}"
+          echo "Failed tests:"
+          node -e "
+            const results = JSON.parse(require('fs').readFileSync('test-results/test-results.json'));
+            results.tests.filter(t => !t.passed).forEach(test => {
+              console.error('\n❌ ' + test.name);
+              if (test.error) console.error(test.error);
+            });
+          "
+          exit 1
+      - name: Upload partition test results
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: test-results-${{ matrix.partition }}
+          path: ui/test-results/test-results.json
+          retention-days: 90
   finalize:
     needs:
       - pre-test
@@ -88,6 +112,24 @@ jobs:
           jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
+      - name: Download all test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: test-results-*
+          path: test-results
+
+      - name: Combine test results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: node ../scripts/combine-ui-test-results.js
+      - name: Upload combined results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: test-results-${{ github.sha }}
+          path: ui/combined-test-results.json
+          retention-days: 90
+
       - name: finalize
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}

scripts/combine-ui-test-results.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+'use strict';
+const fs = require('fs');
+
+const NUM_PARTITIONS = 4;
+
+function combineResults() {
+  const results = [];
+  let duration = 0;
+  let aggregateSummary = { total: 0, passed: 0, failed: 0 };
+
+  for (let i = 1; i <= NUM_PARTITIONS; i++) {
+    try {
+      const data = JSON.parse(
+        fs.readFileSync(`../test-results/test-results-${i}/test-results.json`).toString()
+      );
+      results.push(...data.tests);
+      duration += data.duration;
+      aggregateSummary.total += data.summary.total;
+      aggregateSummary.passed += data.summary.passed;
+      aggregateSummary.failed += data.summary.failed;
+    } catch (err) {
+      console.error(`Error reading partition ${i}:`, err);
+    }
+  }
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    sha: process.env.GITHUB_SHA,
+    summary: {
+      total: aggregateSummary.total,
+      passed: aggregateSummary.passed,
+      failed: aggregateSummary.failed
+    },
+    duration,
+    tests: results
+  };
+
+  fs.writeFileSync('../ui/combined-test-results.json', JSON.stringify(output, null, 2));
+}
+
+if (require.main === module) {
+  combineResults();
+}
+
+module.exports = combineResults;

ui/app/index.html

@@ -24,6 +24,7 @@
     {{content-for "body"}}
 
     <script src="{{rootURL}}assets/vendor.js"></script>
+
     <script src="{{rootURL}}assets/nomad-ui.js"></script>
 
     {{content-for "body-footer"}}

ui/mirage/factories/job.js

@@ -175,6 +175,9 @@ export default Factory.extend({
   // When true, the job will have no versions or deployments (and in turn no latest deployment)
   noDeployments: false,
 
+  // When true, the job will have a previous stable version. Useful for testing "start job" loop.
+  withPreviousStableVersion: false,
+
   // When true, an evaluation with a high modify index and placement failures is created
   failedPlacements: false,
 
@@ -317,8 +320,20 @@ export default Factory.extend({
             version: index,
             noActiveDeployment: job.noActiveDeployment,
             activeDeployment: job.activeDeployment,
+            stable: true,
           });
         });
+
+      if (job.withPreviousStableVersion) {
+        server.create('job-version', {
+          job,
+          namespace: job.namespace,
+          version: 1,
+          noActiveDeployment: job.noActiveDeployment,
+          activeDeployment: job.activeDeployment,
+          stable: true,
+        });
+      }
     }
 
     if (job.activeDeployment) {