Single graph traversal: collect app and page entries and construct a …

…single graph (#73231)

Entry collection is modeled after
https://github.com/vercel/next.js/blob/6f679eef0addac257da440c38d3a5303a763d506/crates/next-api/src/global_module_id_strategy.rs#L19-L23

crates/next-api/src/app.rs

@@ -934,6 +934,7 @@ impl AppEndpoint {
                 let reduced_graphs = get_reduced_graphs_for_page(
                     *rsc_entry,
                     Vc::upcast(this.app_project.client_module_context()),
+                    this.app_project.project(),
                 );
                 // "app/client.js [app-ssr] (ecmascript)" ->
                 //      [("./dynamic", "app/dynamic.js [app-client] (ecmascript)")])]

crates/next-api/src/dynamic_imports.rs

@@ -466,7 +466,8 @@ pub async fn map_next_dynamic(
                 };
                 if is_ssr {
                     if let Some(v) =
-                        &*build_dynamic_imports_map_for_module(client_asset_context, module).await?
+                        &*build_dynamic_imports_map_for_module(client_asset_context, *module)
+                            .await?
                     {
                         return Ok(Some(v.await?.clone_value()));
                     }

crates/next-api/src/module_graph.rs

@@ -1,55 +1,64 @@
 use std::collections::{HashMap, HashSet};
 
-use anyhow::{Context, Result};
-use next_core::next_client_reference::{find_server_entries, ServerEntries};
+use anyhow::Result;
+use next_core::{
+    mode::NextMode,
+    next_client_reference::{find_server_entries, ServerEntries},
+};
 use petgraph::graph::{DiGraph, NodeIndex};
 use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
 use turbopack_core::{
-    context::AssetContext, module::Module, reference::primary_referenced_modules,
+    context::AssetContext,
+    module::{Module, Modules},
+    reference::primary_referenced_modules,
 };
 
-use crate::dynamic_imports::{map_next_dynamic, DynamicImportsHashMap};
+use crate::{
+    dynamic_imports::{map_next_dynamic, DynamicImportsHashMap},
+    project::Project,
+};
 
 #[turbo_tasks::value(cell = "new", eq = "manual", into = "new")]
 #[derive(Clone, Debug, Default)]
 pub struct SingleModuleGraph {
     #[turbo_tasks(trace_ignore)]
-    pub graph: DiGraph<Vc<Box<dyn Module>>, ()>,
+    pub graph: DiGraph<ResolvedVc<Box<dyn Module>>, ()>,
 }
 
 #[turbo_tasks::value(transparent)]
 #[derive(Clone, Debug)]
-pub struct SingleModuleGraphs(pub Vec<Vc<SingleModuleGraph>>);
+pub struct SingleModuleGraphs(pub Vec<ResolvedVc<SingleModuleGraph>>);
 
 #[turbo_tasks::value(transparent)]
 #[derive(Clone, Debug)]
-pub struct ModuleSet(pub HashSet<Vc<Box<dyn Module>>>);
+pub struct ModuleSet(pub HashSet<ResolvedVc<Box<dyn Module>>>);
 
 #[turbo_tasks::value_impl]
 impl SingleModuleGraph {
     #[turbo_tasks::function]
-    pub async fn new_with_entries(entries: Vec<Vc<Box<dyn Module>>>) -> Result<Vc<Self>> {
+    pub async fn new_with_entries(entries: Vc<Modules>) -> Result<Vc<Self>> {
         let mut graph = DiGraph::new();
 
         let mut modules: HashMap<Vc<Box<dyn Module>>, NodeIndex<u32>> = HashMap::new();
-        let mut stack: Vec<_> = entries.into_iter().map(|e| (None, e)).collect();
+        let mut stack: Vec<_> = entries.await?.iter().map(|e| (None, *e)).collect();
         while let Some((parent_idx, module)) = stack.pop() {
             if let Some(idx) = modules.get(&module) {
-                let parent_idx = parent_idx.context("Existing module without parent")?;
-                graph.add_edge(parent_idx, *idx, ());
+                if let Some(parent_idx) = parent_idx {
+                    graph.add_edge(parent_idx, *idx, ());
+                }
                 continue;
             }
 
             let idx = graph.add_node(module);
-            modules.insert(module, idx);
+            modules.insert(*module, idx);
             if let Some(parent_idx) = parent_idx {
                 graph.add_edge(parent_idx, idx, ());
             }
 
             // TODO this includes
             // [project]/packages/next/dist/shared/lib/lazy-dynamic/loadable.js.map
-            for reference in primary_referenced_modules(module).await?.iter() {
-                stack.push((Some(idx), **reference));
+            for reference in primary_referenced_modules(*module).await?.iter() {
+                stack.push((Some(idx), *reference));
             }
         }
 
@@ -58,14 +67,14 @@ impl SingleModuleGraph {
 
     #[turbo_tasks::function]
     pub async fn new_with_entries_visited(
-        entries: Vec<Vc<Box<dyn Module>>>,
+        entries: Vec<ResolvedVc<Box<dyn Module>>>,
         visited_modules: Vc<ModuleSet>,
     ) -> Result<Vc<Self>> {
         let visited_modules = visited_modules.await?;
 
         let mut graph = DiGraph::new();
 
-        let mut modules: HashMap<Vc<Box<dyn Module>>, NodeIndex<u32>> = HashMap::new();
+        let mut modules: HashMap<ResolvedVc<Box<dyn Module>>, NodeIndex<u32>> = HashMap::new();
         let mut stack: Vec<_> = entries.into_iter().map(|e| (None, e)).collect();
         while let Some((parent_idx, module)) = stack.pop() {
             if visited_modules.contains(&module) {
@@ -86,8 +95,8 @@ impl SingleModuleGraph {
 
             // TODO this includes
             // [project]/packages/next/dist/shared/lib/lazy-dynamic/loadable.js.map
-            for reference in primary_referenced_modules(module).await?.iter() {
-                stack.push((Some(idx), **reference));
+            for reference in primary_referenced_modules(*module).await?.iter() {
+                stack.push((Some(idx), *reference));
             }
         }
 
@@ -111,7 +120,9 @@ pub async fn get_module_graph_for_page(
     let graph = SingleModuleGraph::new_with_entries_visited(
         server_utils.clone(),
         Vc::cell(Default::default()),
-    );
+    )
+    .to_resolved()
+    .await?;
     let mut visited_modules: HashSet<_> = graph.await?.graph.node_weights().copied().collect();
 
     let mut graphs = vec![graph];
@@ -123,7 +134,9 @@ pub async fn get_module_graph_for_page(
         let graph = SingleModuleGraph::new_with_entries_visited(
             vec![module],
             Vc::cell(visited_modules.clone()),
-        );
+        )
+        .to_resolved()
+        .await?;
         visited_modules.extend(graph.await?.graph.node_weights().copied());
         graphs.push(graph);
     }
@@ -227,19 +240,35 @@ pub async fn get_reduced_graphs_for_page(
     rsc_entry: Vc<Box<dyn Module>>,
     // TODO instead do that later on per-page traversal
     client_asset_context: Vc<Box<dyn AssetContext>>,
+    project: Vc<Project>,
 ) -> Result<Vc<ReducedGraphs>> {
+    let graphs = match &*project.next_mode().await? {
+        NextMode::Development => get_module_graph_for_page(rsc_entry).await?.0,
+        NextMode::Build => vec![
+            create_module_graph_for_entries(project.get_all_entries())
+                .to_resolved()
+                .await?,
+        ],
+    };
     // if production
     //   // ignore rsc_entry
     //   let graphs = create_module_graph_for_entries(project.get_all_entries())
     // else
     //
-    let graphs = get_module_graph_for_page(rsc_entry).await?.0;
-
     let next_dynamic = graphs
         .iter()
-        .map(|graph| NextDynamicGraph::new_with_entries(*graph, client_asset_context).to_resolved())
+        .map(|graph| {
+            NextDynamicGraph::new_with_entries(**graph, client_asset_context).to_resolved()
+        })
         .try_join()
         .await?;
 
     Ok(ReducedGraphs { next_dynamic }.cell())
 }
+
+#[turbo_tasks::function]
+pub async fn create_module_graph_for_entries(
+    entries: Vc<Modules>,
+) -> Result<Vc<SingleModuleGraph>> {
+    Ok(SingleModuleGraph::new_with_entries(entries))
+}

crates/next-api/src/project.rs

@@ -47,7 +47,7 @@ use turbopack_core::{
     diagnostics::DiagnosticExt,
     file_source::FileSource,
     issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
-    module::Modules,
+    module::{Module, Modules},
     output::{OutputAsset, OutputAssets},
     resolve::{find_context_file, FindContextFileResult},
     source_map::OptionSourceMap,
@@ -67,7 +67,6 @@ use crate::{
     global_module_id_strategy::GlobalModuleIdStrategyBuilder,
     instrumentation::InstrumentationEndpoint,
     middleware::MiddlewareEndpoint,
-    module_graph::SingleModuleGraph,
     pages::PagesProject,
     route::{Endpoint, Route},
     versioned_content_map::{OutputAssetsOperation, VersionedContentMap},
@@ -667,6 +666,72 @@ impl Project {
         get_client_compile_time_info(self.browserslist_query.clone(), self.define_env.client())
     }
 
+    #[turbo_tasks::function]
+    pub async fn get_all_entries(self: Vc<Self>) -> Result<Vc<Modules>> {
+        let mut modules = Vec::new();
+
+        async fn add_endpoint(
+            endpoint: Vc<Box<dyn Endpoint>>,
+            modules: &mut Vec<ResolvedVc<Box<dyn Module>>>,
+        ) -> Result<()> {
+            let root_modules = endpoint.root_modules().await?;
+            modules.extend(root_modules.iter().copied());
+            Ok(())
+        }
+
+        modules.extend(self.client_main_modules().await?.iter().copied());
+
+        let entrypoints = self.entrypoints().await?;
+
+        modules.extend(self.client_main_modules().await?.iter().copied());
+        add_endpoint(entrypoints.pages_error_endpoint, &mut modules).await?;
+        add_endpoint(entrypoints.pages_app_endpoint, &mut modules).await?;
+        add_endpoint(entrypoints.pages_document_endpoint, &mut modules).await?;
+
+        if let Some(middleware) = &entrypoints.middleware {
+            add_endpoint(middleware.endpoint, &mut modules).await?;
+        }
+
+        if let Some(instrumentation) = &entrypoints.instrumentation {
+            let node_js = instrumentation.node_js;
+            let edge = instrumentation.edge;
+            add_endpoint(node_js, &mut modules).await?;
+            add_endpoint(edge, &mut modules).await?;
+        }
+
+        for (_, route) in entrypoints.routes.iter() {
+            match route {
+                Route::Page {
+                    html_endpoint,
+                    data_endpoint,
+                } => {
+                    add_endpoint(*html_endpoint, &mut modules).await?;
+                    add_endpoint(*data_endpoint, &mut modules).await?;
+                }
+                Route::PageApi { endpoint } => {
+                    add_endpoint(*endpoint, &mut modules).await?;
+                }
+                Route::AppPage(page_routes) => {
+                    for page_route in page_routes {
+                        add_endpoint(page_route.html_endpoint, &mut modules).await?;
+                        add_endpoint(page_route.rsc_endpoint, &mut modules).await?;
+                    }
+                }
+                Route::AppRoute {
+                    original_name: _,
+                    endpoint,
+                } => {
+                    add_endpoint(*endpoint, &mut modules).await?;
+                }
+                Route::Conflict => {
+                    tracing::info!("WARN: conflict");
+                }
+            }
+        }
+
+        Ok(Vc::cell(modules))
+    }
+
     #[turbo_tasks::function]
     pub(super) async fn server_compile_time_info(self: Vc<Self>) -> Result<Vc<CompileTimeInfo>> {
         let this = self.await?;
@@ -1317,26 +1382,9 @@ impl Project {
         Ok(Vc::cell(modules))
     }
 
-    #[turbo_tasks::function]
-    pub async fn get_module_graph(self: Vc<Self>) -> Result<Vc<()>> {
-        let mut _single_module_graph = SingleModuleGraph::new_with_entries(
-            self.client_main_modules()
-                .await?
-                .iter()
-                .map(|m| **m)
-                .collect(),
-        )
-        .await?;
-
-        // dbg!(_single_module_graph);
-
-        Ok(Vc::cell(()))
-    }
-
     /// Gets the module id strategy for the project.
     #[turbo_tasks::function]
     pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
-        self.get_module_graph().await?;
         let module_id_strategy = self.next_config().module_id_strategy_config();
         match *module_id_strategy.await? {
             Some(ModuleIdStrategyConfig::Named) => Ok(Vc::upcast(DevModuleIdStrategy::new())),