implement basic side effects optimization (vercel/turborepo#6590)

### Description

* optimizes reexports for modules that are inside of folders with a
package.json that contains `"sideEffects": false`
* For named reexports
* skips resolving, reading, parsing, transforms, code generation and
bundling for unrelated reexports
* skips code generation and bundling for the module containing the
followed reexport
* For star reexports
* skips code generation and bundling for unrelated reexports before the
followed reexport
* skips resolving, reading, parsing, transforms, code generation and
bundling for unrelated reexports after the followed reexport
* skips code generation and bundling for the module containing the
followed reexport


Follow-up work:
* `sideEffects` in package.json also supports a glob. We need to support
that too.
* renaming exports is not yet supported. We need to create a naming
module for that to avoid effects on the importer side.

Closes PACK-2042

crates/node-file-trace/src/lib.rs

@@ -213,10 +213,13 @@ async fn add_glob_results(
     for entry in result.results.values() {
         if let DirectoryEntry::File(path) = entry {
             let source = Vc::upcast(FileSource::new(*path));
-            list.push(asset_context.process(
-                source,
-                Value::new(turbopack_core::reference_type::ReferenceType::Undefined),
-            ));
+            let module = asset_context
+                .process(
+                    source,
+                    Value::new(turbopack_core::reference_type::ReferenceType::Undefined),
+                )
+                .module();
+            list.push(module);
         }
     }
     for result in result.inner.values() {
@@ -259,10 +262,13 @@ async fn input_to_modules(
     for input in input {
         if exact {
             let source = Vc::upcast(FileSource::new(root.join(input)));
-            list.push(asset_context.process(
-                source,
-                Value::new(turbopack_core::reference_type::ReferenceType::Undefined),
-            ));
+            let module = asset_context
+                .process(
+                    source,
+                    Value::new(turbopack_core::reference_type::ReferenceType::Undefined),
+                )
+                .module();
+            list.push(module);
         } else {
             let glob = Glob::new(input);
             add_glob_results(asset_context, root.read_glob(glob, false), &mut list).await?;

crates/turbopack-cli/src/contexts.rs

@@ -5,6 +5,7 @@ use turbo_tasks::{Value, Vc};
 use turbo_tasks_fs::{FileSystem, FileSystemPath};
 use turbopack::{
     condition::ContextCondition,
+    ecmascript::TreeShakingMode,
     module_options::{CustomEcmascriptTransformPlugins, JsxTransformOptions, ModuleOptionsContext},
     resolve_options_context::ResolveOptionsContext,
     ModuleAssetContext,
@@ -99,6 +100,7 @@ async fn get_client_module_options_context(
     let module_options_context = ModuleOptionsContext {
         preset_env_versions: Some(env),
         execution_context: Some(execution_context),
+        tree_shaking_mode: Some(TreeShakingMode::ReexportsOnly),
         ..Default::default()
     };
 

crates/turbopack-core/src/chunk/evaluate.rs

@@ -41,14 +41,16 @@ async fn to_evaluatable(
     asset: Vc<Box<dyn Source>>,
     asset_context: Vc<Box<dyn AssetContext>>,
 ) -> Result<Vc<Box<dyn EvaluatableAsset>>> {
-    let asset = asset_context.process(
-        asset,
-        Value::new(ReferenceType::Entry(EntryReferenceSubType::Runtime)),
-    );
-    let Some(entry) = Vc::try_resolve_downcast::<Box<dyn EvaluatableAsset>>(asset).await? else {
+    let module = asset_context
+        .process(
+            asset,
+            Value::new(ReferenceType::Entry(EntryReferenceSubType::Runtime)),
+        )
+        .module();
+    let Some(entry) = Vc::try_resolve_downcast::<Box<dyn EvaluatableAsset>>(module).await? else {
         bail!(
             "{} is not a valid evaluated entry",
-            asset.ident().to_string().await?
+            module.ident().to_string().await?
         )
     };
     Ok(entry)

crates/turbopack-core/src/context.rs

@@ -1,3 +1,4 @@
+use anyhow::{bail, Result};
 use turbo_tasks::{Value, Vc};
 use turbo_tasks_fs::FileSystemPath;
 
@@ -9,34 +10,70 @@ use crate::{
     source::Source,
 };
 
+#[turbo_tasks::value(shared)]
+pub enum ProcessResult {
+    /// A module was created.
+    Module(Vc<Box<dyn Module>>),
+
+    /// Reference is ignored. This should lead to no module being included by
+    /// the reference.
+    Ignore,
+}
+
+#[turbo_tasks::value_impl]
+impl ProcessResult {
+    #[turbo_tasks::function]
+    pub async fn module(&self) -> Result<Vc<Box<dyn Module>>> {
+        match *self {
+            ProcessResult::Module(m) => Ok(m),
+            ProcessResult::Ignore => {
+                bail!("Expected process result to be a module, but it was ignored")
+            }
+        }
+    }
+}
+
 /// A context for building an asset graph. It's passed through the assets while
 /// creating them. It's needed to resolve assets and upgrade assets to a higher
 /// type (e. g. from FileSource to ModuleAsset).
 #[turbo_tasks::value_trait]
 pub trait AssetContext {
+    /// Gets the compile time info of the asset context.
     fn compile_time_info(self: Vc<Self>) -> Vc<CompileTimeInfo>;
+
+    /// Gets the layer of the asset context.
     fn layer(self: Vc<Self>) -> Vc<String>;
+
+    /// Gets the resolve options for a given path.
     fn resolve_options(
         self: Vc<Self>,
         origin_path: Vc<FileSystemPath>,
         reference_type: Value<ReferenceType>,
     ) -> Vc<ResolveOptions>;
+
+    /// Resolves an request to an [ModuleResolveResult].
     fn resolve_asset(
         self: Vc<Self>,
         origin_path: Vc<FileSystemPath>,
         request: Vc<Request>,
         resolve_options: Vc<ResolveOptions>,
         reference_type: Value<ReferenceType>,
     ) -> Vc<ModuleResolveResult>;
+
+    /// Process a source into a module.
     fn process(
         self: Vc<Self>,
         asset: Vc<Box<dyn Source>>,
         reference_type: Value<ReferenceType>,
-    ) -> Vc<Box<dyn Module>>;
+    ) -> Vc<ProcessResult>;
+
+    /// Process an [ResolveResult] into an [ModuleResolveResult].
     fn process_resolve_result(
         self: Vc<Self>,
         result: Vc<ResolveResult>,
         reference_type: Value<ReferenceType>,
     ) -> Vc<ModuleResolveResult>;
+
+    /// Gets a new AssetContext with the transition applied.
     fn with_transition(self: Vc<Self>, transition: String) -> Vc<Box<dyn AssetContext>>;
 }

crates/turbopack-core/src/ident.rs

@@ -83,6 +83,29 @@ impl ValueToString for AssetIdent {
             s.push(')');
         }
 
+        if let Some(part) = self.part {
+            match &*part.await? {
+                ModulePart::ModuleEvaluation => {
+                    write!(s, " {{module-evaluation}}")?;
+                }
+                ModulePart::Export(export) => {
+                    write!(s, " {{export {}}}", &export.await?)?;
+                }
+                ModulePart::Internal(id) => {
+                    write!(s, " {{internal {}}}", &id)?;
+                }
+                ModulePart::Locals => {
+                    write!(s, " {{locals}}")?;
+                }
+                ModulePart::Reexports => {
+                    write!(s, " {{reexports}}")?;
+                }
+                ModulePart::Facade => {
+                    write!(s, " {{facade}}")?;
+                }
+            }
+        }
+
         Ok(Vc::cell(s))
     }
 }
@@ -245,6 +268,15 @@ impl AssetIdent {
                     3_u8.deterministic_hash(&mut hasher);
                     id.deterministic_hash(&mut hasher);
                 }
+                ModulePart::Locals => {
+                    4_u8.deterministic_hash(&mut hasher);
+                }
+                ModulePart::Reexports => {
+                    5_u8.deterministic_hash(&mut hasher);
+                }
+                ModulePart::Facade => {
+                    6_u8.deterministic_hash(&mut hasher);
+                }
             }
 
             has_hash = true;

crates/turbopack-core/src/reference_type.rs

@@ -36,6 +36,8 @@ pub enum CommonJsReferenceSubType {
 #[derive(Debug, Default, Clone, PartialOrd, Ord, Hash)]
 pub enum EcmaScriptModulesReferenceSubType {
     ImportPart(Vc<ModulePart>),
+    Import,
+    DynamicImport,
     Custom(u8),
     #[default]
     Undefined,
@@ -84,6 +86,7 @@ pub enum EntryReferenceSubType {
     AppRoute,
     AppClientComponent,
     Middleware,
+    Instrumentation,
     Runtime,
     Custom(u8),
     Undefined,
@@ -98,6 +101,7 @@ pub enum ReferenceType {
     Url(UrlReferenceSubType),
     TypeScript(TypeScriptReferenceSubType),
     Entry(EntryReferenceSubType),
+    Runtime,
     Internal(Vc<InnerAssets>),
     Custom(u8),
     Undefined,
@@ -116,6 +120,7 @@ impl Display for ReferenceType {
             ReferenceType::Url(_) => "url",
             ReferenceType::TypeScript(_) => "typescript",
             ReferenceType::Entry(_) => "entry",
+            ReferenceType::Runtime => "runtime",
             ReferenceType::Internal(_) => "internal",
             ReferenceType::Custom(_) => todo!(),
             ReferenceType::Undefined => "undefined",
@@ -154,6 +159,7 @@ impl ReferenceType {
                 matches!(other, ReferenceType::Entry(_))
                     && matches!(sub_type, EntryReferenceSubType::Undefined)
             }
+            ReferenceType::Runtime => matches!(other, ReferenceType::Runtime),
             ReferenceType::Internal(_) => matches!(other, ReferenceType::Internal(_)),
             ReferenceType::Custom(_) => {
                 todo!()
@@ -168,7 +174,9 @@ impl ReferenceType {
     pub fn is_internal(&self) -> bool {
         matches!(
             self,
-            ReferenceType::Internal(_) | ReferenceType::Css(CssReferenceSubType::Internal)
+            ReferenceType::Internal(_)
+                | ReferenceType::Css(CssReferenceSubType::Internal)
+                | ReferenceType::Runtime
         )
     }
 }

crates/turbopack-core/src/resolve/mod.rs

@@ -343,7 +343,6 @@ impl ModuleResolveResultOption {
 #[derive(Clone, Debug)]
 pub enum ResolveResultItem {
     Source(Vc<Box<dyn Source>>),
-    OriginalReferenceExternal,
     OriginalReferenceTypeExternal(String),
     Ignore,
     Empty,
@@ -501,7 +500,7 @@ impl ResolveResult {
     ) -> Result<ModuleResolveResult>
     where
         A: Fn(Vc<Box<dyn Source>>) -> AF,
-        AF: Future<Output = Result<Vc<Box<dyn Module>>>>,
+        AF: Future<Output = Result<ModuleResolveResultItem>>,
         R: Fn(Vc<Box<dyn Source>>) -> RF,
         RF: Future<Output = Result<Vc<Box<dyn ModuleReference>>>>,
     {
@@ -514,12 +513,7 @@ impl ResolveResult {
                     let asset_fn = &source_fn;
                     async move {
                         Ok(match item {
-                            ResolveResultItem::Source(source) => {
-                                ModuleResolveResultItem::Module(Vc::upcast(asset_fn(source).await?))
-                            }
-                            ResolveResultItem::OriginalReferenceExternal => {
-                                ModuleResolveResultItem::OriginalReferenceExternal
-                            }
+                            ResolveResultItem::Source(source) => asset_fn(source).await?,
                             ResolveResultItem::OriginalReferenceTypeExternal(s) => {
                                 ModuleResolveResultItem::OriginalReferenceTypeExternal(s)
                             }
@@ -552,7 +546,11 @@ impl ResolveResult {
         Ok(
             self.await?
                 .map_module(
-                    |asset| async move { Ok(Vc::upcast(RawModule::new(asset))) },
+                    |asset| async move {
+                        Ok(ModuleResolveResultItem::Module(Vc::upcast(RawModule::new(
+                            asset,
+                        ))))
+                    },
                     |source| async move {
                         Ok(Vc::upcast(AffectingResolvingAssetReference::new(source)))
                     },
@@ -839,9 +837,12 @@ pub async fn find_context_file(
     }
     if refs.is_empty() {
         // Tailcall
-        Ok(find_context_file(lookup_path.parent(), names))
+        Ok(find_context_file(
+            lookup_path.parent().resolve().await?,
+            names,
+        ))
     } else {
-        let parent_result = find_context_file(lookup_path.parent(), names).await?;
+        let parent_result = find_context_file(lookup_path.parent().resolve().await?, names).await?;
         Ok(match &*parent_result {
             FindContextFileResult::Found(p, r) => {
                 refs.extend(r.iter().copied());
@@ -2051,6 +2052,7 @@ pub async fn handle_resolve_error(
     })
 }
 
+// TODO this should become a TaskInput instead of a Vc
 /// ModulePart represents a part of a module.
 ///
 /// Currently this is used only for ESMs.
@@ -2063,6 +2065,13 @@ pub enum ModulePart {
     Export(Vc<String>),
     /// A pointer to a specific part.
     Internal(u32),
+    /// The local declarations of a module.
+    Locals,
+    /// The reexports of a module and a reexport of the Locals part.
+    Reexports,
+    /// A facade of the module behaving like the original, but referencing
+    /// internal parts.
+    Facade,
 }
 
 #[turbo_tasks::value_impl]
@@ -2079,4 +2088,16 @@ impl ModulePart {
     pub fn internal(id: u32) -> Vc<Self> {
         ModulePart::Internal(id).cell()
     }
+    #[turbo_tasks::function]
+    pub fn locals() -> Vc<Self> {
+        ModulePart::Locals.cell()
+    }
+    #[turbo_tasks::function]
+    pub fn reexports() -> Vc<Self> {
+        ModulePart::Reexports.cell()
+    }
+    #[turbo_tasks::function]
+    pub fn facade() -> Vc<Self> {
+        ModulePart::Facade.cell()
+    }
 }

crates/turbopack-core/src/resolve/options.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, future::Future, pin::Pin};
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
 use turbo_tasks::{
     debug::ValueDebugFormat, trace::TraceRawVcs, TryJoinIterExt, Value, ValueToString, Vc,
@@ -272,11 +272,14 @@ async fn import_mapping_to_result(
     Ok(match &*mapping.await? {
         ImportMapping::Direct(result) => ImportMapResult::Result(*result),
         ImportMapping::External(name) => ImportMapResult::Result(
-            ResolveResult::primary(name.as_ref().map_or_else(
-                || ResolveResultItem::OriginalReferenceExternal,
-                |req| ResolveResultItem::OriginalReferenceTypeExternal(req.to_string()),
-            ))
-            .into(),
+            ResolveResult::primary(if let Some(name) = name {
+                ResolveResultItem::OriginalReferenceTypeExternal(name.to_string())
+            } else if let Some(request) = request.await?.request() {
+                ResolveResultItem::OriginalReferenceTypeExternal(request)
+            } else {
+                bail!("Cannot resolve external reference without request")
+            })
+            .cell(),
         ),
         ImportMapping::Ignore => {
             ImportMapResult::Result(ResolveResult::primary(ResolveResultItem::Ignore).into())