diff --git a/packages/core/src/utils/exec-promise.ts b/packages/core/src/utils/exec-promise.ts
index 2408eb6dc..5d9f701a1 100644
--- a/packages/core/src/utils/exec-promise.ts
+++ b/packages/core/src/utils/exec-promise.ts
@@ -81,7 +81,13 @@ export default async function execPromise(
       } else {
         // Tools can occasionally print to stderr but not fail, so print that just in case.
         if (allStderr.length) {
-          log.log.warn(allStderr);
+          if (allStderr.includes("Failed to replace env in config")) {
+            const error = new Error(allStderr);
+            error.stack = (error.stack || "") + callSite;
+            reject(error);
+          } else {
+            log.log.warn(allStderr);
+          }
         }
 
         // Resolve the string of the whole stdout
diff --git a/plugins/npm/__tests__/npm-next.test.ts b/plugins/npm/__tests__/npm-next.test.ts
index 1ffeeb57f..2a4338540 100644
--- a/plugins/npm/__tests__/npm-next.test.ts
+++ b/plugins/npm/__tests__/npm-next.test.ts
@@ -262,6 +262,7 @@ describe("next", () => {
       },
     ]);
     execPromise.mockResolvedValueOnce("");
+    execPromise.mockResolvedValueOnce("");
     execPromise.mockResolvedValueOnce("1.2.4-next.0");
 
     plugin.apply(({
@@ -330,6 +331,7 @@ describe("next", () => {
     ]);
     // isMonorepo
     execPromise.mockResolvedValueOnce("");
+    execPromise.mockResolvedValueOnce("");
     execPromise.mockResolvedValueOnce("1.2.4-next.0");
 
     plugin.apply(({
diff --git a/plugins/npm/__tests__/npm.test.ts b/plugins/npm/__tests__/npm.test.ts
index 00c360f2f..576868595 100644
--- a/plugins/npm/__tests__/npm.test.ts
+++ b/plugins/npm/__tests__/npm.test.ts
@@ -824,7 +824,7 @@ describe("canary", () => {
       canaryIdentifier: "canary.123.1",
     });
     expect(execPromise.mock.calls[1]).toContain("npm");
-    expect(execPromise.mock.calls[1][1]).toContain("1.2.4-canary.123.1.0");
+    expect(execPromise.mock.calls[2][1]).toContain("1.2.4-canary.123.1.0");
   });
 
   test("prints canary version in dry run", async () => {
@@ -924,6 +924,7 @@ describe("canary", () => {
 
     // first version exists
     execPromise.mockReturnValueOnce(true);
+    execPromise.mockReturnValueOnce(true);
     // second doesn't
     execPromise.mockReturnValueOnce(false);
 
@@ -931,8 +932,8 @@ describe("canary", () => {
       bump: Auto.SEMVER.patch,
       canaryIdentifier: "canary.123.1",
     });
-    expect(execPromise.mock.calls[2]).toContain("npm");
-    expect(execPromise.mock.calls[2][1]).toContain("1.2.4-canary.123.1.1");
+    expect(execPromise.mock.calls[3]).toContain("npm");
+    expect(execPromise.mock.calls[3][1]).toContain("1.2.4-canary.123.1.1");
   });
 
   test("legacy auth work", async () => {
@@ -1000,8 +1001,8 @@ describe("canary", () => {
       bump: Auto.SEMVER.patch,
       canaryIdentifier: "canary.123.1",
     });
-    expect(execPromise.mock.calls[1]).toContain("npm");
-    expect(execPromise.mock.calls[1][1]).toContain("1.2.4-canary.123.1.0");
+    expect(execPromise.mock.calls[2]).toContain("npm");
+    expect(execPromise.mock.calls[2][1]).toContain("1.2.4-canary.123.1.0");
   });
 
   test("use lerna for monorepo package", async () => {
@@ -1040,7 +1041,7 @@ describe("canary", () => {
       bump: Auto.SEMVER.patch,
       canaryIdentifier: "",
     });
-    expect(execPromise.mock.calls[1][1]).toContain("lerna");
+    expect(execPromise.mock.calls[2][1]).toContain("lerna");
     // @ts-ignore
     expect(value.newVersion).toBe("1.2.3-canary.0");
   });
diff --git a/plugins/npm/package.json b/plugins/npm/package.json
index 005883313..b05460c96 100644
--- a/plugins/npm/package.json
+++ b/plugins/npm/package.json
@@ -41,6 +41,7 @@
     "@auto-it/core": "link:../../packages/core",
     "@auto-it/package-json-utils": "link:../../packages/package-json-utils",
     "await-to-js": "^2.1.1",
+    "endent": "^2.0.1",
     "env-ci": "^5.0.1",
     "fp-ts": "^2.5.3",
     "get-monorepo-packages": "^1.1.0",
diff --git a/plugins/npm/src/index.ts b/plugins/npm/src/index.ts
index 0c5e38c3e..2acae069c 100644
--- a/plugins/npm/src/index.ts
+++ b/plugins/npm/src/index.ts
@@ -1,6 +1,7 @@
 import envCi from "env-ci";
 import * as fs from "fs";
 import path from "path";
+import endent from "endent";
 import { Memoize as memoize } from "typescript-memoize";
 import { RestEndpointMethodTypes } from "@octokit/rest";
 import * as t from "io-ts";
@@ -983,7 +984,7 @@ export default class NPMPlugin implements IPlugin {
       this.name,
       async ({ bump, canaryIdentifier, dryRun, quiet }) => {
         if (this.setRcToken) {
-          await setTokenOnCI(auto.logger);
+          await this.setTokenOnCI(auto);
           auto.logger.verbose.info("Set CI NPM_TOKEN");
         }
 
@@ -1181,7 +1182,7 @@ export default class NPMPlugin implements IPlugin {
       this.name,
       async (preReleaseVersions, { bump, dryRun }) => {
         if (this.setRcToken) {
-          await setTokenOnCI(auto.logger);
+          await this.setTokenOnCI(auto);
           auto.logger.verbose.info("Set CI NPM_TOKEN");
         }
 
@@ -1360,7 +1361,7 @@ export default class NPMPlugin implements IPlugin {
       }
 
       if (this.setRcToken) {
-        await setTokenOnCI(auto.logger);
+        await this.setTokenOnCI(auto);
         auto.logger.verbose.info("Set CI NPM_TOKEN");
       }
 
@@ -1495,4 +1496,31 @@ export default class NPMPlugin implements IPlugin {
       }
     });
   }
+
+  /** The the NPM token */
+  private async setTokenOnCI(auto: Auto) {
+    try {
+      await setTokenOnCI(auto.logger);
+      // This will make NPM actually check if the npmrc is valid for the env
+      await execPromise("npm", ["root"]);
+    } catch (error) {
+      if (
+        // eslint-disable-next-line no-template-curly-in-string
+        error.message?.includes("Failed to replace env in config: ${NPM_TOKEN}")
+      ) {
+        auto.logger.log.error(endent`
+          Uh oh! It looks like you don\'t have a NPM_TOKEN available in your environment.
+
+          To fix:
+
+          - Ensure you've added a NPM_TOKEN environment variable
+          - Ensure that it's exposed to your CI step
+        `);
+        auto.logger.verbose.error(error);
+        process.exit(1);
+      } else {
+        throw error;
+      }
+    }
+  }
 }