Turbopack: Implement regex support for matching webpack loaders (#78733)

Inspired by https://x.com/wSokra/status/1907794635914612777, this allows you to declare a ‘marker’ in place of a glob, allowing you to express more conditions than simply just a glob pattern. To start, this allows us to implement regexes. For example, to match either `.svg` or `.svgr` files, use this `next.config.ts`:

```
const nextConfig: NextConfig = {
  turbopack: {
    rules: {
      '#svg': {
        as: '*.js',
        loaders: ['@svgr/webpack'],
      }
    },
    conditions: {
      '#svg': {
        path: /\.svgr?$/,
      }
    }
  }
};
```

Author: wbinnssmith

Cargo.lock

@@ -37,6 +37,7 @@ remove_console = "0.37.0"
 itertools = { workspace = true }
 auto-hash-map = { workspace = true }
 percent-encoding = "2.3.1"
+serde_path_to_error = { workspace = true }
 
 swc_core = { workspace = true, features = [
   "base",
@@ -60,6 +61,7 @@ modularize_imports = { workspace = true }
 swc_relay = { workspace = true }
 
 turbo-rcstr = { workspace = true }
+turbo-esregex = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-bytes = { workspace = true }
 turbo-tasks-env = { workspace = true }

crates/next-core/Cargo.toml

@@ -10,7 +10,10 @@ use turbo_tasks::{
 use turbo_tasks_env::EnvMap;
 use turbo_tasks_fs::FileSystemPath;
 use turbopack::module_options::{
-    module_options_context::MdxTransformOptions, LoaderRuleItem, OptionWebpackRules,
+    module_options_context::{
+        ConditionItem, ConditionPath, MdxTransformOptions, OptionWebpackConditions,
+    },
+    LoaderRuleItem, OptionWebpackRules,
 };
 use turbopack_core::{
     issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString},
@@ -541,11 +544,57 @@ pub struct TurbopackConfig {
     /// This option has been replaced by `rules`.
     pub loaders: Option<JsonValue>,
     pub rules: Option<FxIndexMap<RcStr, RuleConfigItemOrShortcut>>,
+    #[turbo_tasks(trace_ignore)]
+    pub conditions: Option<FxIndexMap<RcStr, ConfigConditionItem>>,
     pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
     pub resolve_extensions: Option<Vec<RcStr>>,
     pub module_ids: Option<ModuleIds>,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, TraceRawVcs, NonLocalValue)]
+pub struct ConfigConditionItem(ConditionItem);
+
+impl<'de> Deserialize<'de> for ConfigConditionItem {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        struct RegexComponents {
+            source: RcStr,
+            flags: RcStr,
+        }
+
+        #[derive(Deserialize)]
+        struct ConfigPath {
+            path: RegexOrGlob,
+        }
+
+        #[derive(Deserialize)]
+        #[serde(tag = "type", rename_all = "lowercase")]
+        enum RegexOrGlob {
+            Regexp { value: RegexComponents },
+            Glob { value: String },
+        }
+
+        let config_path = ConfigPath::deserialize(deserializer)?;
+        let condition_item = match config_path.path {
+            RegexOrGlob::Regexp { value } => {
+                let regex = turbo_esregex::EsRegex::new(&value.source, &value.flags)
+                    .map_err(serde::de::Error::custom)?;
+                ConditionItem {
+                    path: ConditionPath::Regex(regex.resolved_cell()),
+                }
+            }
+            RegexOrGlob::Glob { value } => ConditionItem {
+                path: ConditionPath::Glob(value.into()),
+            },
+        };
+
+        Ok(ConfigConditionItem(condition_item))
+    }
+}
+
 #[derive(
     Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue, OperationValue,
 )]
@@ -1065,7 +1114,8 @@ impl NextConfig {
     #[turbo_tasks::function]
     pub async fn from_string(string: Vc<RcStr>) -> Result<Vc<Self>> {
         let string = string.await?;
-        let config: NextConfig = serde_json::from_str(&string)
+        let mut jdeserializer = serde_json::Deserializer::from_str(&string);
+        let config: NextConfig = serde_path_to_error::deserialize(&mut jdeserializer)
             .with_context(|| format!("failed to parse next.config.js: {}", string))?;
         Ok(config.cell())
     }
@@ -1227,6 +1277,22 @@ impl NextConfig {
         Vc::cell(Some(ResolvedVc::cell(rules)))
     }
 
+    #[turbo_tasks::function]
+    pub fn webpack_conditions(&self) -> Vc<OptionWebpackConditions> {
+        let Some(config_conditions) = self.turbopack.as_ref().and_then(|t| t.conditions.as_ref())
+        else {
+            return Vc::cell(None);
+        };
+
+        let conditions = FxIndexMap::from_iter(
+            config_conditions
+                .iter()
+                .map(|(k, v)| (k.clone(), v.0.clone())),
+        );
+
+        Vc::cell(Some(ResolvedVc::cell(conditions)))
+    }
+
     #[turbo_tasks::function]
     pub fn persistent_caching_enabled(&self) -> Result<Vc<bool>> {
         Ok(Vc::cell(

crates/next-core/src/next_config.rs

@@ -15,19 +15,22 @@ pub async fn webpack_loader_options(
     project_path: ResolvedVc<FileSystemPath>,
     next_config: Vc<NextConfig>,
     foreign: bool,
-    conditions: Vec<RcStr>,
+    condition_strs: Vec<RcStr>,
 ) -> Result<Option<ResolvedVc<WebpackLoadersOptions>>> {
-    let rules = *next_config.webpack_rules(conditions).await?;
+    let rules = *next_config.webpack_rules(condition_strs).await?;
     let rules = *maybe_add_sass_loader(next_config.sass_config(), rules.map(|v| *v)).await?;
     let rules = if foreign {
         rules
     } else {
         *maybe_add_babel_loader(*project_path, rules.map(|v| *v)).await?
     };
+
+    let conditions = next_config.webpack_conditions().to_resolved().await?;
     Ok(if let Some(rules) = rules {
         Some(
             WebpackLoadersOptions {
                 rules,
+                conditions,
                 loader_runner_package: Some(loader_runner_package_mapping().to_resolved().await?),
             }
             .resolved_cell(),

crates/next-core/src/next_shared/webpack_rules/mod.rs

@@ -874,6 +874,33 @@ function bindingToApi(
       }
     }
 
+    const conditions: (typeof nextConfig)['turbopack']['conditions'] =
+      nextConfigSerializable.turbopack?.conditions
+    if (conditions) {
+      type SerializedConditions = {
+        [key: string]: {
+          path:
+            | { type: 'regexp'; value: { source: string; flags: string } }
+            | { type: 'glob'; value: string }
+        }
+      }
+
+      const serializedConditions: SerializedConditions = {}
+      for (const [key, value] of Object.entries(conditions)) {
+        serializedConditions[key] = {
+          ...value,
+          path:
+            value.path instanceof RegExp
+              ? {
+                  type: 'regexp',
+                  value: { source: value.path.source, flags: value.path.flags },
+                }
+              : { type: 'glob', value: value.path },
+        }
+      }
+      nextConfigSerializable.turbopack.conditions = serializedConditions
+    }
+
     return JSON.stringify(nextConfigSerializable, null, 2)
   }
 

packages/next/src/build/swc/index.ts

@@ -13,6 +13,7 @@ import type {
   TurbopackRuleConfigItem,
   TurbopackRuleConfigItemOptions,
   TurbopackRuleConfigItemOrShortcut,
+  TurbopackRuleCondition,
 } from './config-shared'
 import type {
   Header,
@@ -128,8 +129,13 @@ const zTurboRuleConfigItem: zod.ZodType<TurbopackRuleConfigItem> = z.union([
 const zTurboRuleConfigItemOrShortcut: zod.ZodType<TurbopackRuleConfigItemOrShortcut> =
   z.union([z.array(zTurboLoaderItem), zTurboRuleConfigItem])
 
+const zTurboCondition: zod.ZodType<TurbopackRuleCondition> = z.object({
+  path: z.union([z.string(), z.instanceof(RegExp)]),
+})
+
 const zTurbopackConfig: zod.ZodType<TurbopackOptions> = z.strictObject({
   rules: z.record(z.string(), zTurboRuleConfigItemOrShortcut).optional(),
+  conditions: z.record(z.string(), zTurboCondition).optional(),
   resolveAlias: z
     .record(
       z.string(),

packages/next/src/server/config-schema.ts

@@ -105,6 +105,10 @@ export type TurbopackLoaderItem =
       options: Record<string, JSONValue>
     }
 
+export type TurbopackRuleCondition = {
+  path: string | RegExp
+}
+
 export type TurbopackRuleConfigItemOrShortcut =
   | TurbopackLoaderItem[]
   | TurbopackRuleConfigItem
@@ -144,6 +148,13 @@ export interface TurbopackOptions {
    */
   rules?: Record<string, TurbopackRuleConfigItemOrShortcut>
 
+  /**
+   * (`next --turbopack` only) A list of conditions to apply when running webpack loaders with Turbopack.
+   *
+   * @see [Turbopack Loaders](https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders)
+   */
+  conditions?: Record<string, TurbopackRuleCondition>
+
   /**
    * The module ID strategy to use for Turbopack.
    * If not set, the default is `'named'` for development and `'deterministic'`

packages/next/src/server/config-shared.ts

@@ -29,6 +29,7 @@ tokio = { workspace = true }
 tracing = { workspace = true }
 
 turbo-rcstr = { workspace = true }
+turbo-esregex = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-env = { workspace = true }
 turbo-tasks-fs = { workspace = true }

turbopack/crates/turbopack/Cargo.toml

@@ -627,16 +627,40 @@ impl ModuleOptions {
                     path.context("need_path in ModuleOptions::new is incorrect")?,
                 )
             };
-            for (glob, rule) in webpack_loaders_options.rules.await?.iter() {
+            for (key, rule) in webpack_loaders_options.rules.await?.iter() {
                 rules.push(ModuleRule::new(
                     RuleCondition::All(vec![
-                        if !glob.contains('/') {
-                            RuleCondition::ResourceBasePathGlob(Glob::new(glob.clone()).await?)
-                        } else {
+                        if key.starts_with("#") {
+                            // This is a custom marker requiring a corresponding condition entry
+                            let conditions = (*webpack_loaders_options.conditions.await?)
+                                .context(
+                                    "Expected a condition entry for the webpack loader rule \
+                                     matching {key}. Create a `conditions` mapping in your \
+                                     next.config.js",
+                                )?
+                                .await?;
+
+                            let condition = conditions.get(key).context(
+                                "Expected a condition entry for the webpack loader rule matching \
+                                 {key}.",
+                            )?;
+
+                            match &condition.path {
+                                ConditionPath::Glob(glob) => RuleCondition::ResourcePathGlob {
+                                    base: execution_context.project_path().await?,
+                                    glob: Glob::new(glob.clone()).await?,
+                                },
+                                ConditionPath::Regex(regex) => {
+                                    RuleCondition::ResourcePathEsRegex(regex.await?)
+                                }
+                            }
+                        } else if key.contains('/') {
                             RuleCondition::ResourcePathGlob {
                                 base: execution_context.project_path().await?,
-                                glob: Glob::new(glob.clone()).await?,
+                                glob: Glob::new(key.clone()).await?,
                             }
+                        } else {
+                            RuleCondition::ResourceBasePathGlob(Glob::new(key.clone()).await?)
                         },
                         RuleCondition::not(RuleCondition::ResourceIsVirtualSource),
                     ]),

turbopack/crates/turbopack/src/module_options/mod.rs

@@ -1,4 +1,7 @@
+use std::fmt::Debug;
+
 use serde::{Deserialize, Serialize};
+use turbo_esregex::EsRegex;
 use turbo_rcstr::RcStr;
 use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, ValueDefault, Vc};
 use turbo_tasks_fs::FileSystemPath;
@@ -31,10 +34,31 @@ pub struct WebpackRules(FxIndexMap<RcStr, LoaderRuleItem>);
 #[turbo_tasks::value(transparent)]
 pub struct OptionWebpackRules(Option<ResolvedVc<WebpackRules>>);
 
+#[derive(Default)]
+#[turbo_tasks::value(transparent)]
+pub struct WebpackConditions(pub FxIndexMap<RcStr, ConditionItem>);
+
+#[derive(Default)]
+#[turbo_tasks::value(transparent)]
+pub struct OptionWebpackConditions(Option<ResolvedVc<WebpackConditions>>);
+
+#[derive(Clone, PartialEq, Eq, Debug, TraceRawVcs, Serialize, Deserialize, NonLocalValue)]
+pub enum ConditionPath {
+    Glob(RcStr),
+    Regex(ResolvedVc<EsRegex>),
+}
+
+#[turbo_tasks::value(shared)]
+#[derive(Clone, Debug)]
+pub struct ConditionItem {
+    pub path: ConditionPath,
+}
+
 #[turbo_tasks::value(shared)]
 #[derive(Clone, Debug)]
 pub struct WebpackLoadersOptions {
     pub rules: ResolvedVc<WebpackRules>,
+    pub conditions: ResolvedVc<OptionWebpackConditions>,
     pub loader_runner_package: Option<ResolvedVc<ImportMapping>>,
 }