ci: ensure saucelabs browsers can load karma test page (angular#35171)

In the past we had connecitivity issues on Saucelabs. Browsers on mobile devices were not able to properly resolve the `localhost` hostname through the tunnel. This is because the device resolves `localhost` or `127.0.0.1` to the actual Saucelabs device, while it should resolve to the tunnel host machine (in our case the CircleCI VM). In the past, we simply disabled the failing devices and re-enabled the devices later. At this point, the Saucelabs team claimed that the connecitivy/proxy issues were fixed. Saucelabs seems to have a process for VMs which ensures that requests to `localhost` / `127.0.0.1` are properly resolved through the tunnel. This process is not very reliable and can cause tests to fail. Related issues have been observed/mentioned in the Saucelabs support docs. e.g. https://support.saucelabs.com/hc/en-us/articles/115002212447-Unable-to-Reach-Application-on-localhost-for-Tests-Run-on-Safari-8-and-9-and-Edge https://support.saucelabs.com/hc/en-us/articles/225106887-Safari-and-Internet-Explorer-Won-t-Load-Website-When-Using-Sauce-Connect-on-Localhost In order to ensure that requests are always resolved through the tunnel, we add our own domain alias in the CircleCI's hosts file, and enforce that it is always resolved through the tunnel (using the `--tunnel-domains` SC flag). Saucelabs devices by default will never resolve this domain/hostname to the actual local Saucelabs device. PR Close angular#35171
asydikov · Feb 6, 2020 · 363e1ab · 363e1ab
1 parent dc5ac88
commit 363e1ab
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 27 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -136,6 +136,27 @@ commands:
             git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true
             git config --global gc.auto 0 || true
 
+  init_saucelabs_environment:
+    description: Sets up a domain that resolves to the local host.
+    steps:
+      - run:
+          name: Preparing environment for running tests on Saucelabs.
+          command: |
+            # For SauceLabs jobs, we set up a domain which resolves to the machine which launched
+            # the tunnel. We do this because devices are sometimes not able to properly resolve
+            # `localhost` or `127.0.0.1` through the SauceLabs tunnel. Using a domain that does not
+            # resolve to anything on SauceLabs VMs ensures that such requests are always resolved
+            # through the tunnel, and resolve to the actual tunnel host machine (i.e. the CircleCI VM).
+            # More context can be found in: https://github.com/angular/angular/pull/35171.
+            setPublicVar SAUCE_LOCALHOST_ALIAS_DOMAIN "angular-ci.local"
+            setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          # Sets up a local domain in the machine's host file that resolves to the local
+          # host. This domain is helpful in Saucelabs tests where devices are not able to
+          # properly resolve `localhost` or `127.0.0.1` through the sauce-connect tunnel.
+          name: Setting up alias domain for local host.
+          command: echo "127.0.0.1 $SAUCE_LOCALHOST_ALIAS_DOMAIN" | sudo tee -a /etc/hosts
+
   # Normally this would be an individual job instead of a command.
   # But startup and setup time for each invidual windows job are high enough to discourage
   # many small jobs, so instead we use a command for setup unless the gain becomes significant.
@@ -295,9 +316,7 @@ jobs:
     steps:
       - custom_attach_workspace
       - init_environment
-      - run:
-          name: Preparing environment for running tests on Saucelabs.
-          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - init_saucelabs_environment
       - run:
           name: Run Bazel tests on Saucelabs
           # See /tools/saucelabs/README.md for more info
@@ -319,9 +338,7 @@ jobs:
     steps:
       - custom_attach_workspace
       - init_environment
-      - run:
-          name: Preparing environment for running tests on Saucelabs.
-          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - init_saucelabs_environment
       - run:
           name: Run Bazel tests on Saucelabs
           # See /tools/saucelabs/README.md for more info
@@ -639,11 +656,7 @@ jobs:
     steps:
       - custom_attach_workspace
       - init_environment
-      - run:
-          name: Preparing environment for running tests on Saucelabs.
-          command: |
-            setPublicVar KARMA_JS_BROWSERS $(node -e 'console.log(require("./browser-providers.conf").sauceAliases.CI_REQUIRED.join(","))')
-            setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - init_saucelabs_environment
       - run:
           name: Starting Saucelabs tunnel service
           command: ./tools/saucelabs/sauce-service.sh run
@@ -655,7 +668,11 @@ jobs:
           # Waiting on ready ensures that we don't run tests too early without Saucelabs not being ready.
           name: Waiting for Saucelabs tunnel to connect
           command: ./tools/saucelabs/sauce-service.sh ready-wait
-      - run: yarn karma start ./karma-js.conf.js --single-run --browsers=${KARMA_JS_BROWSERS}
+      - run:
+          name: Running tests on Saucelabs.
+          command: |
+            browsers=$(node -e 'console.log(require("./browser-providers.conf").sauceAliases.CI_REQUIRED.join(","))')
+            yarn karma start ./karma-js.conf.js --single-run --browsers=${browsers}
       - run:
           name: Stop Saucelabs tunnel service
           command: ./tools/saucelabs/sauce-service.sh stop

diff --git a/.circleci/env.sh b/.circleci/env.sh
@@ -65,6 +65,7 @@ setPublicVar SAUCE_TUNNEL_IDENTIFIER "angular-framework-${CIRCLE_BUILD_NUM}-${CI
 # acquire CircleCI instances for too long if sauceconnect failed, we need a connect timeout.
 setPublicVar SAUCE_READY_FILE_TIMEOUT 120
 
+
 ####################################################################################################
 # Define environment variables for the `angular/components` repo unit tests job.
 ####################################################################################################

diff --git a/karma-js.conf.js b/karma-js.conf.js
@@ -167,6 +167,16 @@ module.exports = function(config) {
     conf.browserStack.tunnelIdentifier = tunnelIdentifier;
   }
 
+  // For SauceLabs jobs, we set up a domain which resolves to the machine which launched
+  // the tunnel. We do this because devices are sometimes not able to properly resolve
+  // `localhost` or `127.0.0.1` through the SauceLabs tunnel. Using a domain that does not
+  // resolve to anything on SauceLabs VMs ensures that such requests are always resolved through
+  // the tunnel, and resolve to the actual tunnel host machine (commonly the CircleCI VMs).
+  // More context can be found in: https://github.com/angular/angular/pull/35171.
+  if (process.env.SAUCE_LOCALHOST_ALIAS_DOMAIN) {
+    conf.hostname = process.env.SAUCE_LOCALHOST_ALIAS_DOMAIN;
+  }
+
   if (process.env.KARMA_WEB_TEST_MODE) {
     // KARMA_WEB_TEST_MODE is used to setup karma to run in
     // SauceLabs or Browserstack

diff --git a/tools/saucelabs/karma-saucelabs.js b/tools/saucelabs/karma-saucelabs.js
@@ -21,22 +21,18 @@ try {
   // KARMA_WEB_TEST_MODE is set which informs /karma-js.conf.js that it should
   // run the test with the karma saucelabs launcher
   process.env['KARMA_WEB_TEST_MODE'] = 'SL_REQUIRED';
+  // Saucelabs parameters read from a temporary file that is created by the `sauce-service`. This
+  // will be `null` if the test runs locally without the `sauce-service` being started.
+  const saucelabsParams = readLocalSauceConnectParams();
   // Setup required SAUCE_* env if they are not already set
   if (!process.env['SAUCE_USERNAME'] || !process.env['SAUCE_ACCESS_KEY'] ||
       !process.env['SAUCE_TUNNEL_IDENTIFIER']) {
-    try {
-      // The following path comes from /tools/saucelabs/sauce-service.sh.
-      // We setup the required saucelabs environment variables here for the karma test
-      // from a json file under /tmp/angular/sauce-service  so that we don't break the
-      // test cache with a changing SAUCE_TUNNEL_IDENTIFIER provided through --test_env
-      const scParams = require('/tmp/angular/sauce-service/sauce-connect-params.json');
-      process.env['SAUCE_USERNAME'] = scParams.SAUCE_USERNAME;
-      process.env['SAUCE_ACCESS_KEY'] = scParams.SAUCE_ACCESS_KEY;
-      process.env['SAUCE_TUNNEL_IDENTIFIER'] = scParams.SAUCE_TUNNEL_IDENTIFIER;
-    } catch (e) {
-      console.error(e.stack || e);
-      console.error(
-          `!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    // We print a helpful error message below if the required Saucelabs parameters have not
+    // been specified in test environment, and the `sauce-service` params file has not been
+    // created either.
+    if (saucelabsParams === null) {
+      console.error(`
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !!! Make sure that you have run "yarn bazel run //tools/saucelabs:sauce_service_setup"
 !!! (or "./tools/saucelabs/sauce-service.sh setup") before the test target. Alternately
 !!! you can provide the required SAUCE_* environment variables (SAUCE_USERNAME, SAUCE_ACCESS_KEY &
@@ -45,6 +41,16 @@ try {
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
       process.exit(1);
     }
+    process.env['SAUCE_USERNAME'] = saucelabsParams.SAUCE_USERNAME;
+    process.env['SAUCE_ACCESS_KEY'] = saucelabsParams.SAUCE_ACCESS_KEY;
+    process.env['SAUCE_TUNNEL_IDENTIFIER'] = saucelabsParams.SAUCE_TUNNEL_IDENTIFIER;
+    process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN;
+  }
+
+  // Pass through the optional `SAUCE_LOCALHOST_ALIAS_DOMAIN` environment variable. The
+  // variable is usually specified on CI, but is not required for testing with Saucelabs.
+  if (!process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] && saucelabsParams !== null) {
+    process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN;
   }
 
   const scStart = `${sauceService} start-ready-wait`;
@@ -60,3 +66,15 @@ try {
   console.error(e.stack || e);
   process.exit(1);
 }
+
+function readLocalSauceConnectParams() {
+  try {
+    // The following path comes from /tools/saucelabs/sauce-service.sh.
+    // We setup the required saucelabs environment variables here for the karma test
+    // from a json file under /tmp/angular/sauce-service  so that we don't break the
+    // test cache with a changing SAUCE_TUNNEL_IDENTIFIER provided through --test_env
+    return require('/tmp/angular/sauce-service/sauce-connect-params.json');
+  } catch {
+    return null;
+  }
+}
diff --git a/tools/saucelabs/sauce-service.sh b/tools/saucelabs/sauce-service.sh
@@ -130,7 +130,7 @@ service-setup-command() {
     @fail "sc binary not found at ${SAUCE_CONNECT}"
   fi
 
-  echo "{ \"SAUCE_USERNAME\": \"${SAUCE_USERNAME}\", \"SAUCE_ACCESS_KEY\": \"${SAUCE_ACCESS_KEY}\", \"SAUCE_TUNNEL_IDENTIFIER\": \"${SAUCE_TUNNEL_IDENTIFIER}\" }" > ${SAUCE_PARAMS_JSON_FILE}
+  echo "{ \"SAUCE_USERNAME\": \"${SAUCE_USERNAME}\", \"SAUCE_ACCESS_KEY\": \"${SAUCE_ACCESS_KEY}\", \"SAUCE_TUNNEL_IDENTIFIER\": \"${SAUCE_TUNNEL_IDENTIFIER}\", \"SAUCE_LOCALHOST_ALIAS_DOMAIN\": \"${SAUCE_LOCALHOST_ALIAS_DOMAIN}\" }" > ${SAUCE_PARAMS_JSON_FILE}
 
   # Command arguments that will be passed to sauce-connect.
   # By default we disable SSL bumping for all requests. This is because SSL bumping is
@@ -147,6 +147,16 @@ service-setup-command() {
     "--user ${SAUCE_USERNAME}"
     # Don't add the --api-key here so we don't echo it out in service-pre-start
   )
+
+  if [[ -n "${SAUCE_LOCALHOST_ALIAS_DOMAIN}" ]]; then
+    # Ensures that requests to the localhost alias domain are always resolved through the tunnel.
+    # This environment variable is usually configured on CI, and refers to a domain that has been
+    # locally configured in the current machine's hosts file (e.g. `/etc/hosts`). The domain should
+    # resolve to the current machine in Saucelabs VMs, so we need to ensure that it is resolved
+    # through the tunnel we going to create.
+    sauce_args+=("--tunnel-domains ${SAUCE_LOCALHOST_ALIAS_DOMAIN}")
+  fi
+
   @echo "Sauce connect will be started with:"
   echo "  ${SAUCE_CONNECT} ${sauce_args[@]}"
   SERVICE_COMMAND="${SAUCE_CONNECT} ${sauce_args[@]} --api-key ${SAUCE_ACCESS_KEY}"
@@ -296,7 +306,7 @@ service-post-stop() {
       fi
       @wait_for "Waiting for start file" "${SERVICE_START_FILE}"
       ${SERVICE_COMMAND}
-    ) >>"${SERVICE_LOG_FILE}" 2>&1 
+    ) >>"${SERVICE_LOG_FILE}" 2>&1
   ) &
   echo $! >"${SERVICE_PID_FILE}"