From 2f1d0369a9530e1f88bd1145ebeda4dab674e5e1 Mon Sep 17 00:00:00 2001
From: Mehul Kar <mehul.kar@vercel.com>
Date: Wed, 7 Jun 2023 23:39:13 -0500
Subject: [PATCH] ci(examples): update examples tests to diff full output

This will help debug examples tests when they fail.

- Switch env vars to CLI flags so they don't leak through
  to the turbo invocations in the actual test and change the output that
  we are not asserting against.
- Enable corepack inside the test and then disable it again at the ened
  of the test.
---
 .github/workflows/test.yml                    | 19 ++++++----
 turborepo-tests/examples/run-example.sh       |  2 +-
 .../examples/tests/npm-non-monorepo.t         | 22 ++++++++++--
 turborepo-tests/examples/tests/npm-with-npm.t | 36 +++++++++++++++++--
 .../examples/tests/npm-with-yarn.t            | 36 +++++++++++++++++--
 turborepo-tests/examples/tests/pnpm-basic.t   | 35 ++++++++++++++++--
 turborepo-tests/examples/tests/pnpm-gatsby.t  | 31 ++++++++++++++--
 .../examples/tests/pnpm-kitchen-sink.t        | 35 ++++++++++++++++--
 .../examples/tests/pnpm-with-svelte.t         | 35 ++++++++++++++++--
 turborepo-tests/examples/tests/setup.sh       |  6 ++++
 .../examples/tests/yarn-non-monorepo.t        | 27 ++++++++++++--
 .../examples/tests/yarn-with-npm.t            | 29 +++++++++++++--
 .../examples/tests/yarn-with-yarn.t           | 29 +++++++++++++--
 13 files changed, 314 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3c262db04742c..9060c7009d15b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -391,6 +391,8 @@ jobs:
         run: |
           mkdir -p `pnpm store path`
 
+      # Disable corepack, actions/setup-node invokes other package managers and
+      # that causes corepack to throw an error, so we disable it first.
       - name: Disable corepack
         shell: bash
         run: corepack disable
@@ -404,12 +406,17 @@ jobs:
 
       - name: Check \"${{ matrix.example }}\" example with \"${{ matrix.manager }}\"
         shell: bash
-        env:
-          FORCE_COLOR: true
-          TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
-          TURBO_TEAM: ${{ vars.TURBO_TEAM }}
-          TURBO_REMOTE_ONLY: true
-        run: turbo run test --filter="turborepo-tests-examples" -- "${{ matrix.example }}" "${{ matrix.manager }}"
+        # Note: using CLI flags instead of env vars because
+        # envs var would apply to the actual tests that exercise turbo also,
+        # making test output non-deterministic.
+        run: turbo run test --color --remote-only --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --filter="turborepo-tests-examples" -- "${{ matrix.example }}" "${{ matrix.manager }}"
+
+      # Re-enable corepack, actions/setup-node invokes other package managers and
+      # that causes corepack to throw an error, so we disable it here. The "Post" step
+      # for actions/setup-node will run at the end of this job, so we need to prep for it.
+      - name: Disable corepack again
+        shell: bash
+        run: corepack disable
 
   js_packages:
     name: JS Package Tests
diff --git a/turborepo-tests/examples/run-example.sh b/turborepo-tests/examples/run-example.sh
index bdc8a940fe68b..9bbcd99cf2c90 100755
--- a/turborepo-tests/examples/run-example.sh
+++ b/turborepo-tests/examples/run-example.sh
@@ -12,7 +12,7 @@ TEST_FILE="tests/$2-$1.t"
 
 if [ -f "$TEST_FILE" ]; then
   echo "Running $TEST_FILE"
-  .cram_env/bin/prysk --shell="$(which bash)" "$TEST_FILE" --keep-tmpdir
+  .cram_env/bin/prysk --shell="$(which bash)" "$TEST_FILE"
 else
   echo "Could not find $TEST_FILE"
   exit 1
diff --git a/turborepo-tests/examples/tests/npm-non-monorepo.t b/turborepo-tests/examples/tests/npm-non-monorepo.t
index f7aff1aa10461..aa5c0a03301f1 100644
--- a/turborepo-tests/examples/tests/npm-non-monorepo.t
+++ b/turborepo-tests/examples/tests/npm-non-monorepo.t
@@ -1,5 +1,23 @@
   $ . ${TESTDIR}/setup.sh non-monorepo npm
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ npx turbo build lint > /dev/null 2>&1
-  $ npx turbo build lint > /dev/null 2>&1
+  $ npx turbo build lint --output-logs=none
+  \xe2\x80\xa2 Running build, lint (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    2 successful, 2 total
+  Cached:    0 cached, 2 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+  $ npx turbo build lint --output-logs=none
+  \xe2\x80\xa2 Running build, lint (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    2 successful, 2 total
+  Cached:    2 cached, 2 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/npm-with-npm.t b/turborepo-tests/examples/tests/npm-with-npm.t
index ad598fc92f70e..d9c61b20bb2f3 100644
--- a/turborepo-tests/examples/tests/npm-with-npm.t
+++ b/turborepo-tests/examples/tests/npm-with-npm.t
@@ -1,5 +1,37 @@
   $ . ${TESTDIR}/setup.sh with-npm npm
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ npm run build lint > /dev/null 2>&1
-  $ npm run build lint > /dev/null 2>&1
+  $ npm run build lint -- --output-logs=none
+  
+  \> build (re)
+  \> turbo run build lint --output-logs=none (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+
+
+  $ npm run build lint -- --output-logs=none
+  
+  \> build (re)
+  \> turbo run build lint --output-logs=none (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
+
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/npm-with-yarn.t b/turborepo-tests/examples/tests/npm-with-yarn.t
index 0959af72cc882..1e0087a13c75b 100644
--- a/turborepo-tests/examples/tests/npm-with-yarn.t
+++ b/turborepo-tests/examples/tests/npm-with-yarn.t
@@ -1,5 +1,37 @@
   $ . ${TESTDIR}/setup.sh with-yarn npm
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ npm run build lint > /dev/null 2>&1
-  $ npm run build lint > /dev/null 2>&1
+  $ npm run build lint -- --output-logs=none
+  
+  \> build (re)
+  \> turbo build lint --output-logs=none (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+
+
+  $ npm run build lint -- --output-logs=none
+  
+  \> build (re)
+  \> turbo build lint --output-logs=none (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
+
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/pnpm-basic.t b/turborepo-tests/examples/tests/pnpm-basic.t
index 860204f6634df..c9941ffbeeb38 100644
--- a/turborepo-tests/examples/tests/pnpm-basic.t
+++ b/turborepo-tests/examples/tests/pnpm-basic.t
@@ -1,5 +1,36 @@
   $ . ${TESTDIR}/setup.sh basic pnpm
+  6.26.1
 # run twice and make sure it works
-  $ pnpm run build lint > /dev/null 2>&1
-  $ pnpm run build lint > /dev/null 2>&1
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-basic.t (re)
+  \> turbo run build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+
+
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-basic.t (re)
+  \> turbo run build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
+
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/pnpm-gatsby.t b/turborepo-tests/examples/tests/pnpm-gatsby.t
index 2a7c7e3df73e9..5d5b6903db8a3 100644
--- a/turborepo-tests/examples/tests/pnpm-gatsby.t
+++ b/turborepo-tests/examples/tests/pnpm-gatsby.t
@@ -1,5 +1,30 @@
   $ . ${TESTDIR}/setup.sh with-gatsby pnpm
+  6.26.1
 # run twice and make sure it works
-  $ pnpm run build lint > /dev/null 2>&1
-  $ pnpm run build lint > /dev/null 2>&1
-  $ git diff
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> with-gatsby@0.0.0 build (.*)/pnpm-gatsby.t (re)
+  \> turbo build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> with-gatsby@0.0.0 build (.*)/pnpm-gatsby.t (re)
+  \> turbo build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    3 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
diff --git a/turborepo-tests/examples/tests/pnpm-kitchen-sink.t b/turborepo-tests/examples/tests/pnpm-kitchen-sink.t
index 8ad910e223a69..2a9cbd67a34e6 100644
--- a/turborepo-tests/examples/tests/pnpm-kitchen-sink.t
+++ b/turborepo-tests/examples/tests/pnpm-kitchen-sink.t
@@ -1,5 +1,36 @@
   $ . ${TESTDIR}/setup.sh kitchen-sink pnpm
+  6.26.1
 # run twice and make sure it works
-  $ pnpm run build lint > /dev/null 2>&1
-  $ pnpm run build lint > /dev/null 2>&1
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-kitchen-sink.t (re)
+  \> turbo build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: admin, api, blog, eslint-config-custom, eslint-config-custom-server, jest-presets, logger, storefront, tsconfig, ui (esc)
+  \xe2\x80\xa2 Running build, lint in 10 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    11 successful, 11 total
+  Cached:    0 cached, 11 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+
+
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-kitchen-sink.t (re)
+  \> turbo build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: admin, api, blog, eslint-config-custom, eslint-config-custom-server, jest-presets, logger, storefront, tsconfig, ui (esc)
+  \xe2\x80\xa2 Running build, lint in 10 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    11 successful, 11 total
+  Cached:    11 cached, 11 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
+
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/pnpm-with-svelte.t b/turborepo-tests/examples/tests/pnpm-with-svelte.t
index de8059c6068d0..f4a8cf471129a 100644
--- a/turborepo-tests/examples/tests/pnpm-with-svelte.t
+++ b/turborepo-tests/examples/tests/pnpm-with-svelte.t
@@ -1,5 +1,36 @@
   $ . ${TESTDIR}/setup.sh with-svelte pnpm
+  6.26.1
 # run twice and make sure it works
-  $ pnpm run build lint > /dev/null 2>&1
-  $ pnpm run build lint > /dev/null 2>&1
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-with-svelte.t (re)
+  \> turbo run build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 4 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+
+
+
+  $ pnpm run build lint -- --output-logs=none
+  
+  \> @ build (.*)/pnpm-with-svelte.t (re)
+  \> turbo run build "lint" "--output-logs=none" (re)
+  
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 4 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+
+
+
   $ git diff
diff --git a/turborepo-tests/examples/tests/setup.sh b/turborepo-tests/examples/tests/setup.sh
index fc25699fc00a8..890b7df10bd87 100644
--- a/turborepo-tests/examples/tests/setup.sh
+++ b/turborepo-tests/examples/tests/setup.sh
@@ -30,18 +30,24 @@ function set_package_manager() {
   cat package.json | jq ".packageManager=\"$1\"" | sponge package.json
 }
 
+# Enable corepack so that when we set the packageManager in package.json it actually makes a diference.
+corepack enable
+
 # Set the packageManger version
 NPM_PACKAGE_MANAGER_VALUE="npm@8.1.2"
 PNPM_PACKAGE_MANAGER_VALUE="pnpm@6.26.1"
 YARN_PACKAGE_MANAGER_VALUE="yarn@1.22.17"
 if [ "$pkgManager" == "npm" ]; then
   set_package_manager "$NPM_PACKAGE_MANAGER_VALUE"
+  npm --version
   npm install > /dev/null
 elif [ "$pkgManager" == "pnpm" ]; then
   set_package_manager "$PNPM_PACKAGE_MANAGER_VALUE"
+  pnpm --version
   pnpm install > /dev/null
 elif [ "$pkgManager" == "yarn" ]; then
   set_package_manager "$YARN_PACKAGE_MANAGER_VALUE"
+  yarn --version
   yarn install > /dev/null
 fi
 
diff --git a/turborepo-tests/examples/tests/yarn-non-monorepo.t b/turborepo-tests/examples/tests/yarn-non-monorepo.t
index bf568c053eda8..24fda1bf518e8 100644
--- a/turborepo-tests/examples/tests/yarn-non-monorepo.t
+++ b/turborepo-tests/examples/tests/yarn-non-monorepo.t
@@ -1,5 +1,28 @@
   $ . ${TESTDIR}/setup.sh non-monorepo yarn
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ npx turbo build lint > /dev/null 2>&1
-  $ npx turbo build lint > /dev/null 2>&1
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Running build, lint (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    2 successful, 2 total
+  Cached:    0 cached, 2 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+  Done in [\.0-9]+m?s\. (re)
+
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Running build, lint (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    2 successful, 2 total
+  Cached:    2 cached, 2 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+  Done in [\.0-9]+m?s\. (re)
   $ git diff
diff --git a/turborepo-tests/examples/tests/yarn-with-npm.t b/turborepo-tests/examples/tests/yarn-with-npm.t
index ccaa44edde1c9..1306dae457283 100644
--- a/turborepo-tests/examples/tests/yarn-with-npm.t
+++ b/turborepo-tests/examples/tests/yarn-with-npm.t
@@ -1,5 +1,30 @@
   $ . ${TESTDIR}/setup.sh with-npm yarn
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ yarn build lint > /dev/null 2>&1
-  $ yarn build lint > /dev/null 2>&1
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+  Done in [\.0-9]+m?s\. (re)
+
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+  Done in [\.0-9]+m?s\. (re)
   $ git diff
diff --git a/turborepo-tests/examples/tests/yarn-with-yarn.t b/turborepo-tests/examples/tests/yarn-with-yarn.t
index 5dc81aead1140..a083911511575 100644
--- a/turborepo-tests/examples/tests/yarn-with-yarn.t
+++ b/turborepo-tests/examples/tests/yarn-with-yarn.t
@@ -1,5 +1,30 @@
   $ . ${TESTDIR}/setup.sh with-yarn yarn
+  \d+\.\d+\.\d+ (re)
+
 # run twice and make sure it works
-  $ yarn turbo build lint > /dev/null 2>&1
-  $ yarn turbo build lint > /dev/null 2>&1
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    0 cached, 5 total
+    Time:\s*[\.0-9ms]+  (re)
+  
+  Done in [\.0-9]+m?s\. (re)
+ 
+  $ yarn turbo build lint --output-logs=none
+  yarn run v\d+\.\d+\.\d+ (re)
+  \$ (.*)node_modules/.bin/turbo build lint --output-logs=none (re)
+  \xe2\x80\xa2 Packages in scope: docs, eslint-config-custom, tsconfig, ui, web (esc)
+  \xe2\x80\xa2 Running build, lint in 5 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  
+   Tasks:    5 successful, 5 total
+  Cached:    5 cached, 5 total
+    Time:\s*[\.0-9ms]+ >>> FULL TURBO (re)
+  
+  Done in [\.0-9]+m?s\. (re)
   $ git diff