[test] Disable BrowserStack for PRs

.circleci/config.yml

@@ -1,6 +1,10 @@
 version: 2.1
 
 parameters:
+  browserstack-force:
+    description: Whether to force browserstack usage. We have limited resources on browserstack so the pipeline might decide to skip browserstack if this parameter isn't set to true.
+    type: boolean
+    default: false
   react-dist-tag:
     description: The dist-tag of react to be used
     type: string
@@ -24,6 +28,7 @@ defaults: &defaults
     # Keep in sync with "Save playwright cache"
     PLAYWRIGHT_BROWSERS_PATH: /tmp/pw-browsers
     # expose it globally otherwise we have to thread it from each job to the install command
+    BROWSERSTACK_FORCE: << pipeline.parameters.browserstack-force >>
     REACT_DIST_TAG: << parameters.react-dist-tag >>
     TEST_GATE: << parameters.test-gate >>
   working_directory: /tmp/material-ui

test/README.md

@@ -151,6 +151,22 @@ Our tests run on different browsers to increase the coverage:
 - [Headless Chrome](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md)
 - Chrome, Firefox, Safari, and Edge thanks to [BrowserStack](https://www.browserstack.com)
 
+##### BrowserStack
+
+We only use BrowserStack for non-PR commits to save ressources.
+Browserstack rarely reports actual issues so we only use it as a stop-gap for releases not merges.
+
+To force a run of BrowserStack on a PR you have to run the pipeline with `browserstack-force` set to `true`.
+For example, you've opened a PR with the number 64209 and now after everything is green you want to make sure the change passes all browsers:
+
+```bash
+curl --request POST \
+  --url https://circleci.com/api/v2/project/gh/mui-org/material-ui/pipeline \
+  --header 'content-type: application/json' \
+  --header 'Circle-Token: $CIRCLE_TOKEN' \
+  --data-raw '{"branch":"pull/64209/head","parameters":{"browserstack-force":true}}'
+```
+
 ### Browser API level
 
 In the end, components are going to be used in a real browser.

test/karma.conf.js

@@ -3,6 +3,7 @@ const path = require('path');
 const webpack = require('webpack');
 
 const CI = Boolean(process.env.CI);
+const isPR = Boolean(process.env.CIRCLE_PR_NUMBER);
 
 let build = `material-ui local ${new Date().toISOString()}`;
 
@@ -15,6 +16,11 @@ if (process.env.CIRCLECI) {
 }
 
 const browserStack = {
+  // |commits in PRs| >> |Merged commits|.
+  // Since we have limited ressources on browserstack we often time out on PRs.
+  // However, browserstack rarely fails with a true-positive so we use it as a stop gap for release not merge.
+  // But always enable it locally since people usually have to explicitly have to expose their browserstack access key anyway.
+  enabled: !CI || !isPR || process.env.BROWSERSTACK_FORCE === 'true',
   username: process.env.BROWSERSTACK_USERNAME,
   accessKey: process.env.BROWSERSTACK_ACCESS_KEY,
   build,
@@ -153,7 +159,7 @@ module.exports = function setKarmaConfig(config) {
 
   let newConfig = baseConfig;
 
-  if (browserStack.accessKey) {
+  if (browserStack.enabled && browserStack.accessKey) {
     newConfig = {
       ...baseConfig,
       browserStack,