[Turbopack] WIP changes

crates/next-core/src/app_structure.rs

@@ -33,23 +33,14 @@ use crate::{
 #[turbo_tasks::value]
 #[derive(Default, Debug, Clone)]
 pub struct AppDirModules {
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub page: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub layout: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub error: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub global_error: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub loading: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub template: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub not_found: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub default: Option<Vc<FileSystemPath>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub route: Option<Vc<FileSystemPath>>,
     #[serde(skip_serializing_if = "Metadata::is_empty", default)]
     pub metadata: Metadata,
@@ -145,7 +136,6 @@ pub struct Metadata {
     pub twitter: Vec<MetadataWithAltItem>,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub open_graph: Vec<MetadataWithAltItem>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub sitemap: Option<MetadataItem>,
     // The page indicates where the metadata is defined and captured.
     // The steps for capturing metadata (get_directory_tree) and constructing
@@ -154,7 +144,6 @@ pub struct Metadata {
     // the actual path incorrectly with fillMetadataSegment.
     //
     // This is only being used for the static metadata files.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub base_page: Option<AppPage>,
 }
 
@@ -180,11 +169,8 @@ impl Metadata {
 #[turbo_tasks::value]
 #[derive(Default, Clone, Debug)]
 pub struct GlobalMetadata {
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub favicon: Option<MetadataItem>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub robots: Option<MetadataItem>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub manifest: Option<MetadataItem>,
 }
 

crates/next-core/src/next_config.rs

@@ -209,17 +209,17 @@ pub enum OutputType {
 pub enum RouteHas {
     Header {
         key: RcStr,
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default)]
         value: Option<RcStr>,
     },
     Cookie {
         key: RcStr,
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default)]
         value: Option<RcStr>,
     },
     Query {
         key: RcStr,
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default)]
         value: Option<RcStr>,
     },
     Host {
@@ -238,14 +238,14 @@ pub struct HeaderValue {
 #[serde(rename_all = "camelCase")]
 pub struct Header {
     pub source: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub base_path: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub locale: Option<bool>,
     pub headers: Vec<HeaderValue>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub has: Option<Vec<RouteHas>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub missing: Option<Vec<RouteHas>>,
 }
 
@@ -261,13 +261,13 @@ pub enum RedirectStatus {
 pub struct Redirect {
     pub source: String,
     pub destination: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub base_path: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub locale: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub has: Option<Vec<RouteHas>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub missing: Option<Vec<RouteHas>>,
 
     #[serde(flatten)]
@@ -279,13 +279,13 @@ pub struct Redirect {
 pub struct Rewrite {
     pub source: String,
     pub destination: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub base_path: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub locale: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub has: Option<Vec<RouteHas>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub missing: Option<Vec<RouteHas>>,
 }
 
@@ -378,11 +378,11 @@ pub enum ImageFormat {
 #[serde(rename_all = "camelCase")]
 pub struct RemotePattern {
     pub hostname: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub protocol: Option<RemotePatternProtocal>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub port: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub pathname: Option<String>,
 }
 
@@ -468,9 +468,9 @@ pub enum ReactCompilerMode {
 #[derive(Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ReactCompilerOptions {
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub compilation_mode: Option<ReactCompilerMode>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub panic_threshold: Option<RcStr>,
 }
 

crates/next-core/src/next_manifests/mod.rs

@@ -60,13 +60,12 @@ impl Default for MiddlewaresManifest {
 #[serde(rename_all = "camelCase", default)]
 pub struct MiddlewareMatcher {
     // When skipped next.js with fill that during merging.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub regexp: Option<RcStr>,
-    #[serde(skip_serializing_if = "bool_is_true")]
     pub locale: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub has: Option<Vec<RouteHas>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub missing: Option<Vec<RouteHas>>,
     pub original_source: RcStr,
 }
@@ -83,10 +82,6 @@ impl Default for MiddlewareMatcher {
     }
 }
 
-fn bool_is_true(b: &bool) -> bool {
-    *b
-}
-
 #[derive(Serialize, Default, Debug)]
 pub struct EdgeFunctionDefinition {
     pub files: Vec<RcStr>,

crates/next-core/src/next_server_component/server_component_module.rs

@@ -6,7 +6,7 @@ use turbo_tasks::{RcStr, Vc};
 use turbo_tasks_fs::FileSystemPath;
 use turbopack_core::{
     asset::{Asset, AssetContent},
-    chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext},
+    chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext, ModuleIdJs},
     ident::AssetIdent,
     module::{Module, Modules},
     reference::ModuleReferences,
@@ -20,7 +20,6 @@ use turbopack_ecmascript::{
         esm::{EsmExport, EsmExports},
         external_module::IncludeIdentModule,
     },
-    utils::StringifyJs,
 };
 
 use super::server_component_reference::NextServerComponentModuleReference;
@@ -155,7 +154,7 @@ impl EcmascriptChunkItem for NextServerComponentChunkItem {
                 r#"
                     __turbopack_export_namespace__(__turbopack_import__({}));
                 "#,
-                StringifyJs(&module_id),
+                ModuleIdJs(&module_id),
             )
             .into(),
             ..Default::default()

crates/next-custom-transforms/src/transforms/page_static_info/mod.rs

@@ -56,9 +56,9 @@ impl ExportInfoWarning {
 pub struct ExportInfo {
     pub ssr: bool,
     pub ssg: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
     pub runtime: Option<String>,
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(default)]
     pub preferred_region: Vec<String>,
     pub generate_image_metadata: Option<bool>,
     pub generate_sitemaps: Option<bool>,

packages/next/src/build/index.ts

@@ -1419,16 +1419,16 @@ export default async function build(
         await fs.mkdir(path.join(distDir, 'static', buildId), {
           recursive: true,
         })
-        await fs.writeFile(
-          path.join(distDir, 'package.json'),
-          JSON.stringify(
-            {
-              type: 'commonjs',
-            },
-            null,
-            2
-          )
-        )
+        // await fs.writeFile(
+        //   path.join(distDir, 'package.json'),
+        //   JSON.stringify(
+        //     {
+        //       type: 'commonjs',
+        //     },
+        //     null,
+        //     2
+        //   )
+        // )
 
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const entrypointsSubscription = project.entrypointsSubscribe()

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -207,6 +207,7 @@ impl TurboTasksBackendInner {
 
     fn operation_suspend_point(&self, suspend: impl FnOnce() -> AnyOperation) {
         if self.suspending_requested() {
+            let _span = tracing::trace_span!("operation suspended").entered();
             let operation = Arc::new(suspend());
             let mut snapshot_request = self.snapshot_request.lock();
             if snapshot_request.snapshot_requested {
@@ -233,6 +234,7 @@ impl TurboTasksBackendInner {
     pub(crate) fn start_operation(&self) -> OperationGuard<'_> {
         let fetch_add = self.in_progress_operations.fetch_add(1, Ordering::AcqRel);
         if (fetch_add & SNAPSHOT_REQUESTED_BIT) != 0 {
+            let _span = tracing::trace_span!("operation waiting for start").entered();
             let mut snapshot_request = self.snapshot_request.lock();
             if snapshot_request.snapshot_requested {
                 let value = self.in_progress_operations.fetch_sub(1, Ordering::AcqRel) - 1;
@@ -445,6 +447,7 @@ impl TurboTasksBackendInner {
         let (item, listener) =
             CachedDataItem::new_scheduled_with_listener(self.get_task_desc_fn(task_id), note);
         task.add_new(item);
+        let _span = tracing::trace_span!("recompute from reading output").entered();
         turbo_tasks.schedule(task_id);
 
         Ok(Err(listener))
@@ -643,6 +646,7 @@ impl TurboTasksBackendInner {
         // yet.
         let uncompleted_operations = self.backing_storage.uncompleted_operations();
         if !uncompleted_operations.is_empty() {
+            let _span = tracing::trace_span!("continue uncompleted operations").entered();
             let mut ctx = self.execute_context(turbo_tasks);
             for op in uncompleted_operations {
                 op.execute(&mut ctx);

turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs

@@ -315,16 +315,29 @@ impl AggregatedDataUpdate {
     }
 }
 
+#[derive(Default, Debug, Serialize, Deserialize, Clone, Copy)]
+pub struct AggregationUpdateStats {
+    pub processed_jobs: usize,
+    pub update_aggregation_number: usize,
+    pub new_follower: usize,
+    pub lost_follower: usize,
+    pub data_update: usize,
+    pub find_and_schedule_dirty: usize,
+    pub balance: usize,
+}
+
 #[derive(Default, Serialize, Deserialize, Clone)]
 pub struct AggregationUpdateQueue {
     #[serde(skip)]
     jobs: VecDeque<(Span, AggregationUpdateJob)>,
+    pub stats: AggregationUpdateStats,
 }
 
 impl AggregationUpdateQueue {
     pub fn new() -> Self {
         Self {
             jobs: VecDeque::with_capacity(8),
+            stats: AggregationUpdateStats::default(),
         }
     }
 
@@ -348,6 +361,7 @@ impl AggregationUpdateQueue {
     }
 
     pub fn process(&mut self, ctx: &mut ExecuteContext<'_>) -> bool {
+        self.stats.processed_jobs += 1;
         if let Some((span, job)) = self.jobs.pop_front() {
             let span = span.entered();
             let span = &*span;
@@ -1017,9 +1031,13 @@ impl AggregationUpdateQueue {
 
 impl Operation for AggregationUpdateQueue {
     fn execute(mut self, ctx: &mut ExecuteContext<'_>) {
+        let span = tracing::trace_span!("aggregation update queue", stats = tracing::field::Empty)
+            .entered();
         loop {
             ctx.operation_suspend_point(&self);
             if self.process(ctx) {
+                let stats = self.stats;
+                span.record("stats", tracing::field::debug(stats));
                 return;
             }
         }

turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs

@@ -31,15 +31,9 @@
 impl ConnectChildOperation {
     pub fn run(parent_task_id: TaskId, child_task_id: TaskId, mut ctx: ExecuteContext<'_>) {
         let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All);
-        // Quick skip if the child was already connected before
-        if parent_task
-            .remove(&CachedDataItemKey::OutdatedChild {
-                task: child_task_id,
-            })
-            .is_some()
-        {
-            return;
-        }
+        parent_task.remove(&CachedDataItemKey::OutdatedChild {
+            task: child_task_id,
+        });
         if parent_task.add(CachedDataItem::Child {
             task: child_task_id,
             value: (),
@@ -119,6 +113,9 @@
                     }
                 }
             }
+            if should_schedule {
+                ctx.schedule(child_task_id);
+            }
 
             ConnectChildOperation::UpdateAggregation {
                 aggregation_update: queue,
@@ -133,12 +130,28 @@
         loop {
             ctx.operation_suspend_point(&self);
             match self {
-                ConnectChildOperation::UpdateAggregation {
-                    ref mut aggregation_update,
-                } => {
-                    if aggregation_update.process(ctx) {
-                        self = ConnectChildOperation::Done
-                    }
+                ConnectChildOperation::UpdateAggregation { .. } => {
+                    let span = tracing::trace_span!(
+                        "aggregation update queue in connect child",
+                        stats = tracing::field::Empty
+                    )
+                    .entered();
+                    let stats = loop {
+                        ctx.operation_suspend_point(&self);
+                        let ConnectChildOperation::UpdateAggregation {
+                            ref mut aggregation_update,
+                            ..
+                        } = self
+                        else {
+                            unreachable!();
+                        };
+                        if aggregation_update.process(ctx) {
+                            let stats = aggregation_update.stats;
+                            self = ConnectChildOperation::Done;
+                            break stats;
+                        }
+                    };
+                    span.record("stats", tracing::field::debug(stats));
                 }
 
                 ConnectChildOperation::Done => {