diff --git a/packages/gatsby/src/utils/babel-loader.js b/packages/gatsby/src/utils/babel-loader.js
index a9911d91a888c..acbab0ea76df1 100644
--- a/packages/gatsby/src/utils/babel-loader.js
+++ b/packages/gatsby/src/utils/babel-loader.js
@@ -26,6 +26,7 @@ const { getBrowsersList } = require(`./browserslist`)
 
 const customOptionsCache = new Map()
 const configCache = new Map()
+const babelrcFileToCacheKey = new Map()
 
 module.exports = babelLoader.custom(babel => {
   return {
@@ -71,6 +72,18 @@ module.exports = babelLoader.custom(babel => {
         partialConfig.files.forEach(configFilePath => {
           configCacheKey += `_${configFilePath}`
         })
+
+        // after generating configCacheKey add link between babelrc files and cache keys that rely on it
+        // so we can invalidate memoized configs when used babelrc file changes
+        partialConfig.files.forEach(configFilePath => {
+          let cacheKeysToInvalidate = babelrcFileToCacheKey.get(configFilePath)
+          if (!cacheKeysToInvalidate) {
+            cacheKeysToInvalidate = new Set()
+            babelrcFileToCacheKey.set(configFilePath, cacheKeysToInvalidate)
+          }
+
+          cacheKeysToInvalidate.add(configCacheKey)
+        })
       }
 
       let { options } = partialConfig
@@ -138,3 +151,22 @@ module.exports = babelLoader.custom(babel => {
     },
   }
 })
+
+module.exports.BabelConfigItemsCacheInvalidatorPlugin = class BabelConfigItemsCacheInvalidatorPlugin {
+  constructor() {
+    this.name = `BabelConfigItemsCacheInvalidatorPlugin`
+  }
+
+  apply(compiler) {
+    compiler.hooks.invalid.tap(this.name, function (file) {
+      const cacheKeysToInvalidate = babelrcFileToCacheKey.get(file)
+
+      if (cacheKeysToInvalidate) {
+        for (const cacheKey of cacheKeysToInvalidate) {
+          configCache.delete(cacheKey)
+        }
+        babelrcFileToCacheKey.delete(file)
+      }
+    })
+  }
+}
diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index c2d91efa28a9b..233ee7e789d17 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -21,6 +21,7 @@ import { StaticQueryMapper } from "./webpack/static-query-mapper"
 import { ForceCssHMRForEdgeCases } from "./webpack/force-css-hmr-for-edge-cases"
 import { getBrowsersList } from "./browserslist"
 import { builtinModules } from "module"
+const { BabelConfigItemsCacheInvalidatorPlugin } = require(`./babel-loader`)
 
 const FRAMEWORK_BUNDLES = [`react`, `react-dom`, `scheduler`, `prop-types`]
 
@@ -211,6 +212,7 @@ module.exports = async (
       }),
 
       plugins.virtualModules(),
+      new BabelConfigItemsCacheInvalidatorPlugin(),
     ]
 
     switch (stage) {