Skip to content

Commit

Permalink
test(system-tests): support docker-based tests against built binary (#…
Browse files Browse the repository at this point in the history 
…20250)

Co-authored-by: Ryan Manuel <ryanm@cypress.io>
  • Loading branch information
@flotwig @ryanthemanuel
flotwig and ryanthemanuel authored Feb 18, 2022
1 parent 75c8750 commit ad2f4de
Show file tree
Hide file tree
Showing 13 changed files with 504 additions and 128 deletions.
20 changes: 0 additions & 20 deletions __snapshots__/bump-spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,6 @@ exports['list of all projects'] = [
"repo": "cypress-io/cypress-test-module-api",
"provider": "circle",
"platform": "linux"
},
{
"repo": "cypress-io/cypress-test-node-versions",
"provider": "circle",
"platform": "linux"
},
{
"repo": "cypress-io/cypress-test-ci-environments",
"provider": "circle",
"platform": "linux"
}
]

Expand All @@ -21,15 +11,5 @@ exports['should have just circle and linux projects'] = [
"repo": "cypress-io/cypress-test-module-api",
"provider": "circle",
"platform": "linux"
},
{
"repo": "cypress-io/cypress-test-node-versions",
"provider": "circle",
"platform": "linux"
},
{
"repo": "cypress-io/cypress-test-ci-environments",
"provider": "circle",
"platform": "linux"
}
]
77 changes: 56 additions & 21 deletions circle.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ defaults: &defaults
type: boolean
default: false
executor: <<parameters.executor>>
environment:
environment: &defaultsEnvironment
## set specific timezone
TZ: "/usr/share/zoneinfo/America/New_York"

Expand Down Expand Up @@ -169,7 +169,7 @@ commands:
command: node scripts/circle-cache.js --action cacheKey > circle_cache_key
- restore_cache:
name: Restore cache state, to check for known modules cache existence
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
- run:
name: Move node_modules back from /tmp
command: |
Expand All @@ -193,8 +193,8 @@ commands:
- restore_cache:
name: Restore system tests node_modules cache
keys:
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
- v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
- v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-

update_cached_system_tests_deps:
description: 'Update the cached node_modules for projects in "system-tests/projects/**"'
Expand All @@ -205,36 +205,36 @@ commands:
- restore_cache:
name: Restore cache state, to check for known modules cache existence
keys:
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
- v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
- run:
name: Send root honeycomb event for this CI build
command: cd system-tests/scripts && node ./send-root-honecomb-event.js
- run:
name: Bail if specific cache exists
command: |
if [[ -f "system_tests_node_modules_installed" ]]; then
if [[ -f "/tmp/system_tests_node_modules_installed" ]]; then
echo "No updates to system tests node modules, exiting"
circleci-agent step halt
fi
- restore_cache:
name: Restore system tests node_modules cache
keys:
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
- v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
- v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-
- run:
name: Update system-tests node_modules cache
command: yarn workspace @tooling/system-tests projects:yarn:install
- save_cache:
name: Save system tests node_modules cache
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
paths:
- ~/.cache/cy-system-tests-node-modules
- run: touch system_tests_node_modules_installed
- run: touch /tmp/system_tests_node_modules_installed
- save_cache:
name: Save system tests node_modules cache state key
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
paths:
- system_tests_node_modules_installed
- /tmp/system_tests_node_modules_installed

caching-dependency-installer:
description: 'Installs & caches the dependencies based on yarn lock & package json dependencies'
Expand All @@ -249,19 +249,19 @@ commands:
command: node scripts/circle-cache.js --action cacheKey > circle_cache_key
- restore_cache:
name: Restore cache state, to check for known modules cache existence
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-state-{{ checksum "circle_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-state-{{ checksum "circle_cache_key" }}
- run:
name: Bail if cache exists
command: |
if [[ -f "node_modules_installed" ]]; then
if [[ -f "/tmp/node_modules_installed" ]]; then
echo "Node modules already cached for dependencies, exiting"
circleci-agent step halt
fi
- run: date +%Y-%U > cache_date
- restore_cache:
name: Restore weekly yarn cache
keys:
- v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-deps-root-weekly-{{ checksum "cache_date" }}
- v{{ .Environment.CACHE_VERSION }}-deps-root-weekly-{{ checksum "cache_date" }}
- run:
name: Install Node Modules
command: |
Expand All @@ -274,7 +274,7 @@ commands:
steps:
- save_cache:
name: Saving node modules for root, cli, and all globbed workspace packages
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
paths:
- node_modules
- cli/node_modules
Expand All @@ -285,18 +285,18 @@ commands:
steps:
- save_cache:
name: Saving node modules for root, cli, and all globbed workspace packages
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
paths:
- /tmp/node_modules_cache
- run: touch node_modules_installed
- run: touch /tmp/node_modules_installed
- save_cache:
name: Saving node-modules cache state key
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-state-{{ checksum "circle_cache_key" }}
key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-state-{{ checksum "circle_cache_key" }}
paths:
- node_modules_installed
- /tmp/node_modules_installed
- save_cache:
name: Save weekly yarn cache
key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-deps-root-weekly-{{ checksum "cache_date" }}
key: v{{ .Environment.CACHE_VERSION }}-deps-root-weekly-{{ checksum "cache_date" }}
paths:
- ~/.yarn

Expand Down Expand Up @@ -519,6 +519,24 @@ commands:
path: /tmp/artifacts
- store-npm-logs

run-binary-system-tests:
steps:
- restore_cached_workspace
- restore_cached_system_tests_deps
- run:
name: Run system tests
command: |
ALL_SPECS=`circleci tests glob "$HOME/cypress/system-tests/test-binary/*spec*"`
SPECS=`echo $ALL_SPECS | xargs -n 1 | circleci tests split --split-by=timings`
echo SPECS=$SPECS
yarn workspace @tooling/system-tests test:ci $SPECS
- verify-mocha-results
- store_test_results:
path: /tmp/cypress
- store_artifacts:
path: /tmp/artifacts
- store-npm-logs

store-npm-logs:
description: Saves any NPM debug logs as artifacts in case there is a problem
steps:
Expand Down Expand Up @@ -1162,6 +1180,19 @@ jobs:
- restore_cached_workspace
- update_cached_system_tests_deps

binary-system-tests:
parallelism: 2
working_directory: ~/cypress
environment:
<<: *defaultsEnvironment
machine:
# using `machine` gives us a Linux VM that can run Docker
image: ubuntu-2004:202111-02
docker_layer_caching: true
resource_class: medium
steps:
- run-binary-system-tests

system-tests-chrome:
<<: *defaults
resource_class: medium
Expand Down Expand Up @@ -2279,6 +2310,10 @@ linux-workflow: &linux-workflow
name: "test binary as a root user"
requires:
- create-build-artifacts
- binary-system-tests:
requires:
- create-build-artifacts
- system-tests-node-modules-install

mac-workflow: &mac-workflow
jobs:
Expand Down
9 changes: 0 additions & 9 deletions guides/release-process.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,8 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
- [cypress-example-todomvc-redux](https://github.com/cypress-io/cypress-example-todomvc-redux/issues/1)
- [cypress-example-realworld](https://github.com/cypress-io/cypress-example-realworld/issues/2)
- [cypress-example-recipes](https://github.com/cypress-io/cypress-example-recipes/issues/225)
- [cypress-example-api-testing](https://github.com/cypress-io/cypress-example-api-testing/issues/15)
- [angular-pizza-creator](https://github.com/cypress-io/angular-pizza-creator/issues/5)
- [cypress-fiddle](https://github.com/cypress-io/cypress-fiddle/issues/5)
- [cypress-example-piechopper](https://github.com/cypress-io/cypress-example-piechopper/issues/75)
- [cypress-documentation](https://github.com/cypress-io/cypress-documentation/issues/1313)
- [cypress-example-docker-compose](https://github.com/cypress-io/cypress-example-docker-compose) - Doesn't have a Renovate issue, but will auto-create and auto-merge non-major Cypress updates as long as the tests pass.

Expand All @@ -209,11 +207,6 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
**Test Repos**

- [cypress-test-tiny](https://github.com/cypress-io/cypress-test-tiny)
- [cypress-test-nested-projects](https://github.com/cypress-io/cypress-test-nested-projects)
- [cypress-test-example-repos](https://github.com/cypress-io/cypress-test-example-repos)
- [cypress-test-node-versions](https://github.com/cypress-io/cypress-test-node-versions)
- [cypress-test-module-api](https://github.com/cypress-io/cypress-test-module-api)
- [cypress-test-ci-environments](https://github.com/cypress-io/cypress-test-ci-environments)

**Example Repos**

Expand All @@ -222,8 +215,6 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
- [cypress-example-realworld](https://github.com/cypress-io/cypress-example-realworld)
- [cypress-example-recipes](https://github.com/cypress-io/cypress-example-recipes)
- [cypress-example-docker-compose](https://github.com/cypress-io/cypress-example-docker-compose)
- [cypress-example-api-testing](https://github.com/cypress-io/cypress-example-api-testing)
- [cypress-example-piechopper](https://github.com/cypress-io/cypress-example-piechopper)
- [cypress-documentation](https://github.com/cypress-io/cypress-documentation)

Take a break, you deserve it! :sunglasses:
Expand Down
2 changes: 0 additions & 2 deletions scripts/binary/bump.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ const _PROVIDERS = {
main: 'cypress-io/cypress',
linux: [
'cypress-io/cypress-test-module-api',
'cypress-io/cypress-test-node-versions',
'cypress-io/cypress-test-ci-environments',
],
},
}
Expand Down
32 changes: 31 additions & 1 deletion system-tests/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ describe('my new project', () => {
systemTests.setup()

systemTests.it('fails as expected', {
project: Fixtures.projectPath('my-new-project'),
project: 'my-new-project',
snapshot: true,
spec: '*',
expectedExitCode: 2
Expand All @@ -61,6 +61,36 @@ From here, you could run this test with `yarn test my-new-project`.

There are many more options available for `systemTests.it` and `systemTests.setup`. You can massage the stdout, do pre-run tasks, set up HTTP/S servers, and more. Explore the typedocs in [`./lib/system-tests`](./lib/system-tests) for more information.

These tests run in the `system-tests-*` CI jobs.

### Developing Docker-based tests against built binary

Specs in the [`./test`](./test) directory are run against an unbuilt Cypress App. They don't test `cypress` NPM package installation or other prod app behavior. This is done so that they can run as fast as possible in CI, without waiting for a full build of the Cypress App.

Specs in [`./test-binary`](./test-binary) are run against the *built Cypress App*. They also run inside of their own Docker containers to give a blank slate environment for Cypress to run in. Before each test, the prod CLI is `npm install`ed along with the built Cypress `.zip`, and real `cypress run` commands are used to run the tests. There should be no functional difference between running a project in these tests and running real prod Cypress inside of Docker in CI.

The purpose of these tests is to test things that we normally can't inside of regular `system-tests`, such as testing Cypress with different Node versions, with/without Xvfb, or inside of different operating system versions.

An example of using `dockerImage` and `withBinary` to write a binary system test:

```ts
// ./test-binary/node-versions.spec.ts
import systemTests from '../lib/system-tests'
import Fixtures from '../lib/fixtures'

describe('node versions', () => {
systemTests.it('runs in node 12', {
dockerImage: 'cypress:node/12',
project: 'todos',
withBinary: true,
})
})
```

Running `yarn test node-versions` would spin up a local Docker container for `cypress:node/12`, install Cypress from `../cypress.zip` and `../cli/build`, and then call the regular `cypress run` command within the container. Other options for `systemTests.it` such as `onRun` and `expectedExitCode` still function normally.

These tests run in the `binary-system-tests` CI job.

### Updating Snaphots

Prepend `SNAPSHOT_UPDATE=1` to any test command. See [`snap-shot-it` instructions](https://github.com/bahmutov/snap-shot-it#advanced-use) for more info.
Expand Down
Loading

1 comment on commit ad2f4de

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on ad2f4de Feb 18, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.5.1/linux-x64/circle-develop-ad2f4de84412f675021d02409d30e1c5722a988d/cypress.tgz

Please sign in to comment.