test: Migrate more runner UI specs to app (#20118)

* Adding remaining reporter error tests.

* Removing existing integration spec file

* Removing existing runner fixtures for error specs

* Fixing ts lint

* Testing with less tests kept in memory

* Testing some cypress-in-cypress settings to improve run-mode performance.

* Reverting changes to doc_url specs

* PR Updates. Inverting default value for 'open in IDE' validation in an attempt to reduce test time.

* Cleaning up ported code

* Update system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html

Co-authored-by: Emily Rohrbough  <emilyrohrbough@users.noreply.github.com>

* Trying build workaround

* Bumping up parallelized builds for app integration tests

* Removing suspicious tests to validate CI build

* Removing more suspicious tests to test CI

* Trying to not scaffold every navigation

* Bumping resources for run-app-integration job. Updating reporter.errors specs to appropriately reduce AUT refreshes.

* Couple API tweaks

* test: Migrating runner hooks specs to app

* Removing unused helper from runner package; cleaning up a few things

* Updating yarn.lock

* fix: Migrating runner sessions ui tests to app

* test: Migrating runner's runner.ui tests to app

* test: Migrating runner retries ui tests to app

* Cleaning up a few things for consistency

* Fixing ts errors

* Removing tests for runner header that have existing coverage in app

* Working around detached elements in CI

* Still working around detached elements in CI

* Updating sessions.ui test to click more carefully. Adding bit more coverage around runner header and snapshot pinning.

* Update packages/app/cypress/e2e/runner/retries.ui.cy.ts

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

* Migrating issue specs. Updating TODOs. Hopefully addressing some flake.

* Stopping icon spin animation for percy snapshots

* WIP

* customPercyCommand.ts

* WIP: working on custom percy snapshots for hiding elements other than the reporter [skip ci]

* WIP: Adding naive mutation observer for reseting DOM mutations after snapshots [skip ci]

Got tired of tripping over types, I can add them when we're feature complete

* Implementing scoped snapshots throughout app/runner tests, let's see how percy output looks

* Adding more snapshots to runner tests.

* Using spec link as click target

* Stop taking snapshots of failed tests

* Scoping selector playground snapshots (though they're still unstyled)

* Addressing flake in virtualized spec list

* Drying up runner snapshotting

* Logging single percy snapshot that reflects proper viewport and overrides

* Updating describes to better match previous snapshot names

* Removing added snapshots to get more accurate better diff presentation in Percy

* Removing new runner.ui snapshots as well

* Adding a few PR recommendations. Updating one last spec name that was corrected for grammar and no longer matched. Updating/removing relevant TODOs.

* Updating to use anticipated default width for reporter

* Updating tests for consistency with previous snapshots

* Tying original command into promise chain

* Let's try that again

* Making percy command enqueuing explicit to test result

* Scoping down on some variables. Using defaults for sessions tests.

* Updating some overrides to see if visibility is handled inconsistently

* Whitespace vanquished, working session test updates back in for long command logs

* Viewport height is not reflected in snapshots, let's try forcing the height on the panel

* Forcing the height issue

* Let's try a minHeight on the percy command

* Cleaning up alterations made during percy exploration

* Trying to get a cleaner percy diff after merge

* Fix Percy to use 10.0-release branch as baseline image comparison

* add comments about temporary percy env vars

* Adding some doc for changes made to customPercyCommand

Co-authored-by: Emily Rohrbough  <emilyrohrbough@users.noreply.github.com>
Co-authored-by: Ryan Manuel <ryanm@cypress.io>
Co-authored-by: Brian Mann <brian.mann86@gmail.com>
Co-authored-by: Emily Rohrbough <emilyrohrbough@yahoo.com>

circle.yml

@@ -457,6 +457,11 @@ commands:
             # https://www.gep13.co.uk/blog/chocolatey-error-hashes-do-not-match
             [[ $PLATFORM == 'windows' && '<<parameters.browser>>' == 'chrome' ]] && choco install googlechrome --ignore-checksums || [[ $PLATFORM != 'windows' ]]
       - run:
+          # the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
+          # are a temporary hack to workaround percy issues with
+          # PR's not diffing the right base branch due to other problems
+          # upstream with finalizing builds. These will be removed
+          # once we implement a more permanent solution.
           command: |
             cmd=$([[ <<parameters.percy>> == 'true' ]] && echo 'yarn percy exec --parallel -- --') || true
             DEBUG=<<parameters.debug>> \
@@ -465,6 +470,8 @@ commands:
             PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 \
             PERCY_ENABLE=${PERCY_TOKEN:-0} \
             PERCY_PARALLEL_TOTAL=-1 \
+            PERCY_TARGET_BRANCH="10.0-release" \
+            PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
             $cmd yarn workspace @packages/<<parameters.package>> cypress:run:<<parameters.type>> --browser <<parameters.browser>> --record --parallel --group <<parameters.package>>-<<parameters.type>>
       - store_test_results:
           path: /tmp/cypress
@@ -1034,7 +1041,17 @@ jobs:
               echo "This is an external PR, cannot access other services"
               circleci-agent step halt
             fi
-      - run: PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 yarn percy build:finalize
+      - run:
+          # the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
+          # are a temporary hack to workaround percy issues with
+          # PR's not diffing the right base branch due to other problems
+          # upstream with finalizing builds. These will be removed
+          # once we implement a more permanent solution.
+          command: |
+            PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 \
+            PERCY_TARGET_BRANCH="10.0-release" \
+            PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
+            yarn percy build:finalize
 
   cli-visual-tests:
     <<: *defaults
@@ -1054,10 +1071,17 @@ jobs:
           path: cli/visual-snapshots
       - run:
           name: Upload CLI snapshots for diffing
+          # the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
+          # are a temporary hack to workaround percy issues with
+          # PR's not diffing the right base branch due to other problems
+          # upstream with finalizing builds. These will be removed
+          # once we implement a more permanent solution.
           command: |
             PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 \
             PERCY_ENABLE=${PERCY_TOKEN:-0} \
             PERCY_PARALLEL_TOTAL=-1 \
+            PERCY_TARGET_BRANCH="10.0-release" \
+            PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
             yarn percy snapshot ./cli/visual-snapshots
 
   unit-tests:
@@ -1325,12 +1349,19 @@ jobs:
           command: yarn build-for-tests
           working_directory: packages/reporter
       - run:
+          # the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
+          # are a temporary hack to workaround percy issues with
+          # PR's not diffing the right base branch due to other problems
+          # upstream with finalizing builds. These will be removed
+          # once we implement a more permanent solution.
           command: |
             CYPRESS_KONFIG_ENV=production \
             CYPRESS_RECORD_KEY=$PACKAGES_RECORD_KEY \
             PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 \
             PERCY_ENABLE=${PERCY_TOKEN:-0} \
             PERCY_PARALLEL_TOTAL=-1 \
+            PERCY_TARGET_BRANCH="10.0-release" \
+            PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
             yarn percy exec --parallel -- -- \
             yarn cypress:run --record --parallel --group reporter
           working_directory: packages/reporter
@@ -1468,11 +1499,19 @@ jobs:
       - run:
           name: Run tests
           # will use PERCY_TOKEN environment variable if available
+          #
+          # the PERCY_TARGET_BRANCH and PERCY_TARGET_COMMIT env vars
+          # are a temporary hack to workaround percy issues with
+          # PR's not diffing the right base branch due to other problems
+          # upstream with finalizing builds. These will be removed
+          # once we implement a more permanent solution.
           command: |
             CYPRESS_KONFIG_ENV=production \
             PERCY_PARALLEL_NONCE=$PLATFORM-$CIRCLE_SHA1 \
             PERCY_ENABLE=${PERCY_TOKEN:-0} \
             PERCY_PARALLEL_TOTAL=-1 \
+            PERCY_TARGET_BRANCH="10.0-release" \
+            PERCY_TARGET_COMMIT=$(git log -n 1 origin/10.0-release --pretty="%H") \
             yarn percy exec --parallel -- -- \
             yarn test --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json
           working_directory: npm/design-system

packages/app/cypress/component/support/index.ts

@@ -24,7 +24,7 @@ import { createPinia } from '../../../src/store'
 import { setActivePinia } from 'pinia'
 import type { Pinia } from 'pinia'
 import 'cypress-real-events/support'
-import installCustomPercyCommand from '@packages/ui-components/cypress/support/customPercyCommand'
+import { installCustomPercyCommand } from '@packages/ui-components/cypress/support/customPercyCommand'
 
 let pinia: Pinia
 

packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts

@@ -1,4 +1,5 @@
 import defaultMessages from '@packages/frontend-shared/src/locales/en-US.json'
+import { snapshotAUTPanel } from './support/snapshot-aut-panel'
 
 describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
   beforeEach(() => {
@@ -23,19 +24,19 @@ describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
     cy.contains('Firefox').should('be.visible')
     cy.findByTestId('viewport').click()
 
-    cy.percySnapshot('browsers open')
+    snapshotAUTPanel('browsers open')
     cy.contains('Firefox').should('be.hidden')
     cy.contains('The viewport determines the width and height of your application. By default the viewport will be 500px by 500px for Component Testing unless specified by a cy.viewport command.')
     .should('be.visible')
 
-    cy.percySnapshot('viewport info open')
+    snapshotAUTPanel('viewport info open')
 
     cy.get('body').click()
 
     cy.findByTestId('playground-activator').click()
     cy.findByTestId('playground-selector').clear().type('#__cy_root')
 
-    cy.percySnapshot('cy.get selector')
+    snapshotAUTPanel('cy.get selector')
 
     cy.findByTestId('playground-num-elements').contains('1 Match')
 
@@ -49,7 +50,7 @@ describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
 
     cy.findByTestId('playground-selector').clear().type('Component Test')
 
-    cy.percySnapshot('cy.contains selector')
+    snapshotAUTPanel('cy.contains selector')
 
     cy.findByTestId('playground-num-elements').contains('1 Match')
   })

packages/app/cypress/e2e/cypress-in-cypress-e2e.cy.ts

@@ -1,4 +1,5 @@
 import defaultMessages from '@packages/frontend-shared/src/locales/en-US.json'
+import { snapshotAUTPanel } from './support/snapshot-aut-panel'
 
 describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
   beforeEach(() => {
@@ -23,19 +24,19 @@ describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
     cy.contains('Firefox').should('be.visible')
     cy.findByTestId('viewport').click()
 
-    cy.percySnapshot('browsers open')
+    snapshotAUTPanel('browsers open')
     cy.contains('Firefox').should('be.hidden')
     cy.contains('The viewport determines the width and height of your application. By default the viewport will be 1000px by 660px for End-to-end Testing unless specified by a cy.viewport command.')
     .should('be.visible')
 
-    cy.percySnapshot('viewport info open')
+    snapshotAUTPanel('viewport info open')
 
     cy.get('body').click()
 
     cy.findByTestId('playground-activator').click()
     cy.findByTestId('playground-selector').clear().type('li')
 
-    cy.percySnapshot('cy.get selector')
+    snapshotAUTPanel('cy.get selector')
 
     cy.findByTestId('playground-num-elements').contains('3 Matches')
 
@@ -44,7 +45,7 @@ describe('Cypress In Cypress', { viewportWidth: 1200 }, () => {
 
     cy.findByTestId('playground-selector').clear().type('Item 1')
 
-    cy.percySnapshot('cy.contains selector')
+    snapshotAUTPanel('cy.contains selector')
 
     cy.findByTestId('playground-num-elements').contains('1 Match')
 

packages/runner/cypress/e2e/issues/issue-18042.js → packages/app/cypress/e2e/runner/issues/issue-18042.cy.js

@@ -1,15 +1,13 @@
-const helpers = require('../../support/helpers')
-
-const { createCypress } = helpers
-const { runIsolatedCypress } = createCypress()
+import { loadSpec } from '../support/spec-loader'
 
 // https://github.com/cypress-io/cypress/issues/18042
 describe('issue 18042', () => {
-  beforeEach(function () {
-    return runIsolatedCypress(`cypress/fixtures/issues/issue-18042.js`)
-  })
-
   it('Call count is shown even if cy.stub().log(false)', function () {
+    loadSpec({
+      fileName: 'issue-18042.cy.js',
+      passCount: 1,
+    })
+
     cy.contains('Spies / Stubs (1)').click()
     cy.get('.call-count').eq(1).should('have.text', '1')
   })

packages/app/cypress/e2e/runner/issues/issue-9162.cy.js

@@ -0,0 +1,14 @@
+import { loadSpec } from '../support/spec-loader'
+
+// https://github.com/cypress-io/cypress/issues/9162
+describe('issue 9162', () => {
+  it('tests does not hang even if there is a fail in before().', function () {
+    loadSpec({
+      fileName: 'issue-9162.cy.js',
+      passCount: 0,
+      failCount: 1,
+    })
+
+    cy.contains('expected true to be false')
+  })
+})