[Turbopack] HMR invalidation improvements (#70494)

### What? * round chunk item sizes to cause less changes to chunking * pick individual values from compile time info to avoid invalidating on change * pick next config keys for more granular caching * split completions into a tree --------- Co-authored-by: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com>
vercel · Oct 2, 2024 · 043dbc8 · 043dbc8
1 parent 1d1b846
commit 043dbc8
Show file tree

Hide file tree

Showing 18 changed files with 224 additions and 109 deletions.
diff --git a/crates/next-core/src/next_client/transforms.rs b/crates/next-core/src/next_client/transforms.rs
@@ -29,9 +29,9 @@ pub async fn get_next_client_transforms_rules(
 ) -> Result<Vec<ModuleRule>> {
     let mut rules = vec![];
 
-    let modularize_imports_config = &next_config.await?.modularize_imports;
+    let modularize_imports_config = &next_config.modularize_imports().await?;
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
-    if let Some(modularize_imports_config) = modularize_imports_config {
+    if !modularize_imports_config.is_empty() {
         rules.push(get_next_modularize_imports_rule(
             modularize_imports_config,
             enable_mdx_rs,

diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs
@@ -36,6 +36,9 @@ struct CustomRoutes {
     rewrites: Vc<Rewrites>,
 }
 
+#[turbo_tasks::value(transparent)]
+pub struct ModularizeImports(IndexMap<String, ModularizeImportPackageConfig>);
+
 #[turbo_tasks::value(serialization = "custom", eq = "manual")]
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -481,7 +484,8 @@ pub enum ReactCompilerOptionsOrBoolean {
 #[turbo_tasks::value(transparent)]
 pub struct OptionalReactCompilerOptions(Option<Vc<ReactCompilerOptions>>);
 
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, TraceRawVcs)]
+#[turbo_tasks::value(eq = "manual")]
+#[derive(Clone, Debug, Default, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct ExperimentalConfig {
     pub allowed_revalidate_header_keys: Option<Vec<RcStr>>,
@@ -723,7 +727,8 @@ impl StyledComponentsTransformOptionsOrBoolean {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TraceRawVcs)]
+#[turbo_tasks::value(eq = "manual")]
+#[derive(Clone, Debug, PartialEq, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct CompilerConfig {
     pub react_remove_properties: Option<ReactRemoveProperties>,
@@ -801,6 +806,11 @@ impl NextConfig {
         )
     }
 
+    #[turbo_tasks::function]
+    pub fn compiler(&self) -> Vc<CompilerConfig> {
+        self.compiler.clone().unwrap_or_default().cell()
+    }
+
     #[turbo_tasks::function]
     pub fn env(&self) -> Vc<EnvMap> {
         // The value expected for env is Record<String, String>, but config itself
@@ -994,6 +1004,16 @@ impl NextConfig {
         options.cell()
     }
 
+    #[turbo_tasks::function]
+    pub fn modularize_imports(&self) -> Vc<ModularizeImports> {
+        Vc::cell(self.modularize_imports.clone().unwrap_or_default())
+    }
+
+    #[turbo_tasks::function]
+    pub fn experimental(&self) -> Vc<ExperimentalConfig> {
+        self.experimental.clone().cell()
+    }
+
     #[turbo_tasks::function]
     pub fn react_compiler(&self) -> Vc<OptionalReactCompilerOptions> {
         let options = &self.experimental.react_compiler;

diff --git a/crates/next-core/src/next_server/transforms.rs b/crates/next-core/src/next_server/transforms.rs
@@ -32,9 +32,9 @@ pub async fn get_next_server_transforms_rules(
 ) -> Result<Vec<ModuleRule>> {
     let mut rules = vec![];
 
-    let modularize_imports_config = &next_config.await?.modularize_imports;
+    let modularize_imports_config = &next_config.modularize_imports().await?;
     let mdx_rs = next_config.mdx_rs().await?.is_some();
-    if let Some(modularize_imports_config) = modularize_imports_config {
+    if !modularize_imports_config.is_empty() {
         rules.push(get_next_modularize_imports_rule(
             modularize_imports_config,
             mdx_rs,

diff --git a/crates/next-core/src/next_shared/transforms/emotion.rs b/crates/next-core/src/next_shared/transforms/emotion.rs
@@ -9,10 +9,10 @@ use crate::next_config::{EmotionTransformOptionsOrBoolean, NextConfig};
 pub async fn get_emotion_transform_rule(next_config: Vc<NextConfig>) -> Result<Option<ModuleRule>> {
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
     let module_rule = next_config
+        .compiler()
         .await?
-        .compiler
+        .emotion
         .as_ref()
-        .and_then(|value| value.emotion.as_ref())
         .and_then(|config| match config {
             EmotionTransformOptionsOrBoolean::Boolean(true) => {
                 EmotionTransformer::new(&Default::default())

diff --git a/crates/next-core/src/next_shared/transforms/modularize_imports.rs b/crates/next-core/src/next_shared/transforms/modularize_imports.rs
@@ -18,7 +18,7 @@ use turbopack_ecmascript::{CustomTransformer, EcmascriptInputTransform, Transfor
 
 use super::module_rule_match_js_no_url;
 
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, TraceRawVcs)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
 #[serde(rename_all = "camelCase")]
 pub struct ModularizeImportPackageConfig {
     pub transform: Transform,
@@ -28,7 +28,7 @@ pub struct ModularizeImportPackageConfig {
     pub skip_default_conversion: bool,
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, TraceRawVcs)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
 #[serde(untagged)]
 pub enum Transform {
     #[default]

diff --git a/crates/next-core/src/next_shared/transforms/react_remove_properties.rs b/crates/next-core/src/next_shared/transforms/react_remove_properties.rs
@@ -21,10 +21,10 @@ pub async fn get_react_remove_properties_transform_rule(
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
 
     let module_rule = next_config
+        .compiler()
         .await?
-        .compiler
+        .react_remove_properties
         .as_ref()
-        .and_then(|value| value.react_remove_properties.as_ref())
         .and_then(|config| match config {
             ReactRemoveProperties::Boolean(false) => None,
             ReactRemoveProperties::Boolean(true) => {

diff --git a/crates/next-core/src/next_shared/transforms/relay.rs b/crates/next-core/src/next_shared/transforms/relay.rs
@@ -14,14 +14,12 @@ pub async fn get_relay_transform_rule(
 ) -> Result<Option<ModuleRule>> {
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
     let project_path = &*project_path.await?;
-    let module_rule = next_config.await?.compiler.as_ref().and_then(|value| {
-        value.relay.as_ref().map(|config| {
-            get_ecma_transform_rule(
-                Box::new(RelayTransformer::new(config, project_path)),
-                enable_mdx_rs,
-                true,
-            )
-        })
+    let module_rule = next_config.compiler().await?.relay.as_ref().map(|config| {
+        get_ecma_transform_rule(
+            Box::new(RelayTransformer::new(config, project_path)),
+            enable_mdx_rs,
+            true,
+        )
     });
 
     Ok(module_rule)

diff --git a/crates/next-core/src/next_shared/transforms/remove_console.rs b/crates/next-core/src/next_shared/transforms/remove_console.rs
@@ -21,10 +21,10 @@ pub async fn get_remove_console_transform_rule(
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
 
     let module_rule = next_config
+        .compiler()
         .await?
-        .compiler
+        .remove_console
         .as_ref()
-        .and_then(|value| value.remove_console.as_ref())
         .and_then(|config| match config {
             RemoveConsoleConfig::Boolean(false) => None,
             RemoveConsoleConfig::Boolean(true) => Some(remove_console::Config::All(true)),

diff --git a/crates/next-core/src/next_shared/transforms/styled_components.rs b/crates/next-core/src/next_shared/transforms/styled_components.rs
@@ -14,10 +14,10 @@ pub async fn get_styled_components_transform_rule(
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
 
     let module_rule = next_config
+        .compiler()
         .await?
-        .compiler
+        .styled_components
         .as_ref()
-        .and_then(|value| value.styled_components.as_ref())
         .and_then(|config| match config {
             StyledComponentsTransformOptionsOrBoolean::Boolean(true) => {
                 Some(StyledComponentsTransformer::new(&Default::default()))

diff --git a/crates/next-core/src/next_shared/transforms/swc_ecma_transform_plugins.rs b/crates/next-core/src/next_shared/transforms/swc_ecma_transform_plugins.rs
@@ -11,7 +11,7 @@ pub async fn get_swc_ecma_transform_plugin_rule(
     next_config: Vc<NextConfig>,
     project_path: Vc<FileSystemPath>,
 ) -> Result<Option<ModuleRule>> {
-    match next_config.await?.experimental.swc_plugins.as_ref() {
+    match next_config.experimental().await?.swc_plugins.as_ref() {
         Some(plugin_configs) if !plugin_configs.is_empty() => {
             #[cfg(feature = "plugin")]
             {

diff --git a/crates/next-core/src/transform_options.rs b/crates/next-core/src/transform_options.rs
@@ -136,12 +136,7 @@ pub async fn get_jsx_transform_options(
         false
     };
 
-    let is_emotion_enabled = next_config
-        .await?
-        .compiler
-        .as_ref()
-        .map(|c| c.emotion.is_some())
-        .unwrap_or_default();
+    let is_emotion_enabled = next_config.compiler().await?.emotion.is_some();
 
     // [NOTE]: ref: WEB-901
     // next.js does not allow to overriding react runtime config via tsconfig /

diff --git a/turbopack/crates/turbo-tasks/src/completion.rs b/turbopack/crates/turbo-tasks/src/completion.rs
@@ -60,14 +60,26 @@ impl Completions {
     /// Merges the list of completions into one.
     #[turbo_tasks::function]
     pub async fn completed(&self) -> anyhow::Result<Vc<Completion>> {
-        self.0
-            .iter()
-            .map(|&c| async move {
-                c.await?;
-                Ok(())
-            })
-            .try_join()
-            .await?;
-        Ok(Completion::new())
+        if self.0.len() > 100 {
+            let mid = self.0.len() / 2;
+            let (left, right) = self.0.split_at(mid);
+            let left = Vc::<Completions>::cell(left.to_vec());
+            let right = Vc::<Completions>::cell(right.to_vec());
+            let left = left.completed();
+            let right = right.completed();
+            left.await?;
+            right.await?;
+            Ok(Completion::new())
+        } else {
+            self.0
+                .iter()
+                .map(|&c| async move {
+                    c.await?;
+                    Ok(())
+                })
+                .try_join()
+                .await?;
+            Ok(Completion::new())
+        }
     }
 }
diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs
@@ -713,6 +713,11 @@ pub trait ChunkType: ValueToString {
     ) -> Vc<usize>;
 }
 
+pub fn round_chunk_item_size(size: usize) -> usize {
+    let a = size.next_power_of_two();
+    size & (a | (a >> 1) | (a >> 2))
+}
+
 #[turbo_tasks::value(transparent)]
 pub struct ChunkItems(Vec<Vc<Box<dyn ChunkItem>>>);
 
@@ -752,3 +757,43 @@ where
         chunk_item.chunking_context().chunk_item_id(chunk_item)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_round_chunk_item_size() {
+        assert_eq!(round_chunk_item_size(0), 0);
+        assert_eq!(round_chunk_item_size(1), 1);
+        assert_eq!(round_chunk_item_size(2), 2);
+        assert_eq!(round_chunk_item_size(3), 3);
+        assert_eq!(round_chunk_item_size(4), 4);
+        assert_eq!(round_chunk_item_size(5), 4);
+        assert_eq!(round_chunk_item_size(6), 6);
+        assert_eq!(round_chunk_item_size(7), 6);
+        assert_eq!(round_chunk_item_size(8), 8);
+
+        assert_eq!(changes_in_range(0..1000), 19);
+        assert_eq!(changes_in_range(1000..2000), 2);
+        assert_eq!(changes_in_range(2000..3000), 1);
+
+        assert_eq!(changes_in_range(3000..10000), 4);
+
+        fn changes_in_range(range: std::ops::Range<usize>) -> usize {
+            let len = range.len();
+            let mut count = 0;
+            for i in range {
+                let a = round_chunk_item_size(i);
+                assert!(a >= i * 2 / 3);
+                assert!(a <= i);
+                let b = round_chunk_item_size(i + 1);
+
+                if a == b {
+                    count += 1;
+                }
+            }
+            len - count
+        }
+    }
+}
diff --git a/turbopack/crates/turbopack-core/src/compile_time_info.rs b/turbopack/crates/turbopack-core/src/compile_time_info.rs
@@ -166,6 +166,12 @@ impl From<String> for DefineableNameSegment {
 #[derive(Debug, Clone)]
 pub struct CompileTimeDefines(pub IndexMap<Vec<DefineableNameSegment>, CompileTimeDefineValue>);
 
+#[turbo_tasks::value(transparent)]
+#[derive(Debug, Clone)]
+pub struct CompileTimeDefinesIndividual(
+    pub IndexMap<Vec<DefineableNameSegment>, Vc<CompileTimeDefineValue>>,
+);
+
 impl IntoIterator for CompileTimeDefines {
     type Item = (Vec<DefineableNameSegment>, CompileTimeDefineValue);
     type IntoIter = indexmap::map::IntoIter<Vec<DefineableNameSegment>, CompileTimeDefineValue>;
@@ -181,6 +187,16 @@ impl CompileTimeDefines {
     pub fn empty() -> Vc<Self> {
         Vc::cell(IndexMap::new())
     }
+
+    #[turbo_tasks::function]
+    pub fn individual(&self) -> Vc<CompileTimeDefinesIndividual> {
+        Vc::cell(
+            self.0
+                .iter()
+                .map(|(key, value)| (key.clone(), value.clone().cell()))
+                .collect(),
+        )
+    }
 }
 
 #[turbo_tasks::value]
@@ -223,12 +239,28 @@ impl From<CompileTimeDefineValue> for FreeVarReference {
 #[derive(Debug, Clone)]
 pub struct FreeVarReferences(pub IndexMap<Vec<DefineableNameSegment>, FreeVarReference>);
 
+#[turbo_tasks::value(transparent)]
+#[derive(Debug, Clone)]
+pub struct FreeVarReferencesIndividual(
+    pub IndexMap<Vec<DefineableNameSegment>, Vc<FreeVarReference>>,
+);
+
 #[turbo_tasks::value_impl]
 impl FreeVarReferences {
     #[turbo_tasks::function]
     pub fn empty() -> Vc<Self> {
         Vc::cell(IndexMap::new())
     }
+
+    #[turbo_tasks::function]
+    pub fn individual(&self) -> Vc<FreeVarReferencesIndividual> {
+        Vc::cell(
+            self.0
+                .iter()
+                .map(|(key, value)| (key.clone(), value.clone().cell()))
+                .collect(),
+        )
+    }
 }
 
 #[turbo_tasks::value(shared)]

diff --git a/turbopack/crates/turbopack-css/src/chunk/mod.rs b/turbopack/crates/turbopack-css/src/chunk/mod.rs
@@ -10,8 +10,8 @@ use turbo_tasks_fs::{rope::Rope, File, FileSystem};
 use turbopack_core::{
     asset::{Asset, AssetContent},
     chunk::{
-        AsyncModuleInfo, Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkType,
-        ChunkableModule, ChunkingContext, ModuleId, OutputChunk, OutputChunkRuntimeInfo,
+        round_chunk_item_size, AsyncModuleInfo, Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo,
+        ChunkType, ChunkableModule, ChunkingContext, ModuleId, OutputChunk, OutputChunkRuntimeInfo,
     },
     code_builder::{Code, CodeBuilder},
     ident::AssetIdent,
@@ -489,12 +489,9 @@ impl ChunkType for CssChunkType {
         else {
             bail!("Chunk item is not an css chunk item but reporting chunk type css");
         };
-        Ok(Vc::cell(
-            chunk_item
-                .content()
-                .await
-                .map_or(0, |content| content.inner_code.len()),
-        ))
+        Ok(Vc::cell(chunk_item.content().await.map_or(0, |content| {
+            round_chunk_item_size(content.inner_code.len())
+        })))
     }
 }