From 609181e429990a36fdbb3b8d243b708535538be6 Mon Sep 17 00:00:00 2001 From: hrmny <8845940+ForsakenHarmony@users.noreply.github.com> Date: Thu, 16 May 2024 02:10:23 +0200 Subject: [PATCH] feat(turbopack-ecmascript): use import attributes for annotations (vercel/turbo#6732) --- crates/turbopack-core/src/reference_type.rs | 7 + .../directives/server_to_client_proxy.rs | 29 +-- .../src/analyzer/imports.rs | 210 +++++++++--------- .../turbopack-ecmascript/src/annotations.rs | 34 +++ crates/turbopack-ecmascript/src/lib.rs | 1 + .../src/references/esm/base.rs | 17 +- .../src/references/mod.rs | 18 +- crates/turbopack-resolve/src/ecmascript.rs | 15 +- crates/turbopack/src/lib.rs | 21 +- 9 files changed, 201 insertions(+), 151 deletions(-) create mode 100644 crates/turbopack-ecmascript/src/annotations.rs diff --git a/crates/turbopack-core/src/reference_type.rs b/crates/turbopack-core/src/reference_type.rs index d8adea7770510..57a0a6722bcd2 100644 --- a/crates/turbopack-core/src/reference_type.rs +++ b/crates/turbopack-core/src/reference_type.rs @@ -33,11 +33,18 @@ pub enum CommonJsReferenceSubType { Undefined, } +#[turbo_tasks::value(serialization = "auto_for_input")] +#[derive(Debug, Clone, PartialOrd, Ord, Hash)] +pub enum ImportWithType { + Json, +} + #[turbo_tasks::value(serialization = "auto_for_input")] #[derive(Debug, Default, Clone, PartialOrd, Ord, Hash)] pub enum EcmaScriptModulesReferenceSubType { ImportPart(Vc), Import, + ImportWithType(ImportWithType), DynamicImport, Custom(u8), #[default] diff --git a/crates/turbopack-ecmascript-plugins/src/transform/directives/server_to_client_proxy.rs b/crates/turbopack-ecmascript-plugins/src/transform/directives/server_to_client_proxy.rs index 1415759e5a783..e9b55df950940 100644 --- a/crates/turbopack-ecmascript-plugins/src/transform/directives/server_to_client_proxy.rs +++ b/crates/turbopack-ecmascript-plugins/src/transform/directives/server_to_client_proxy.rs @@ -2,28 +2,22 @@ use swc_core::{ common::DUMMY_SP, ecma::{ ast::{ - Expr, ExprStmt, Ident, ImportDecl, ImportSpecifier, ImportStarAsSpecifier, - KeyValueProp, Lit, Module, ModuleDecl, ModuleItem, ObjectLit, Program, Prop, PropName, - PropOrSpread, Stmt, Str, + ImportDecl, ImportSpecifier, ImportStarAsSpecifier, Module, ModuleDecl, ModuleItem, + Program, }, utils::private_ident, }, quote, }; -use turbopack_ecmascript::TURBOPACK_HELPER; +use turbopack_ecmascript::{ + annotations::{with_clause, ANNOTATION_TRANSITION}, + TURBOPACK_HELPER, +}; pub fn create_proxy_module(transition_name: &str, target_import: &str) -> Program { let ident = private_ident!("clientProxy"); Program::Module(Module { body: vec![ - ModuleItem::Stmt(Stmt::Expr(ExprStmt { - expr: Box::new(Expr::Lit(Lit::Str(Str { - value: format!("TURBOPACK {{ transition: {transition_name} }}").into(), - raw: None, - span: DUMMY_SP, - }))), - span: DUMMY_SP, - })), ModuleItem::ModuleDecl(ModuleDecl::Import(ImportDecl { specifiers: vec![ImportSpecifier::Namespace(ImportStarAsSpecifier { local: ident.clone(), @@ -31,13 +25,10 @@ pub fn create_proxy_module(transition_name: &str, target_import: &str) -> Progra })], src: Box::new(target_import.into()), type_only: false, - with: Some(Box::new(ObjectLit { - span: DUMMY_SP, - props: vec![PropOrSpread::Prop(Box::new(Prop::KeyValue(KeyValueProp { - key: PropName::Ident(Ident::new(TURBOPACK_HELPER.into(), DUMMY_SP)), - value: Box::new(Expr::Lit(true.into())), - })))], - })), + with: Some(with_clause(&[ + (TURBOPACK_HELPER.as_str(), "true"), + (ANNOTATION_TRANSITION, transition_name), + ])), span: DUMMY_SP, phase: Default::default(), })), diff --git a/crates/turbopack-ecmascript/src/analyzer/imports.rs b/crates/turbopack-ecmascript/src/analyzer/imports.rs index 215cc053aa576..faa84ebbcccee 100644 --- a/crates/turbopack-ecmascript/src/analyzer/imports.rs +++ b/crates/turbopack-ecmascript/src/analyzer/imports.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt::Display, mem::take}; +use std::{collections::BTreeMap, fmt::Display}; use indexmap::{IndexMap, IndexSet}; use once_cell::sync::Lazy; @@ -14,43 +14,76 @@ use turbo_tasks::Vc; use turbopack_core::{issue::IssueSource, source::Source}; use super::{JsValue, ModuleValue}; -use crate::utils::unparen; #[turbo_tasks::value(serialization = "auto_for_input")] #[derive(Default, Debug, Clone, Hash, PartialOrd, Ord)] pub struct ImportAnnotations { // TODO store this in more structured way #[turbo_tasks(trace_ignore)] - map: BTreeMap>, + map: BTreeMap, } -/// Enables a specified transtion for the annotated import -static ANNOTATION_TRANSITION: Lazy = Lazy::new(|| "transition".into()); +/// Enables a specified transition for the annotated import +static ANNOTATION_TRANSITION: Lazy = + Lazy::new(|| crate::annotations::ANNOTATION_TRANSITION.into()); /// Changes the chunking type for the annotated import -static ANNOTATION_CHUNKING_TYPE: Lazy = Lazy::new(|| "chunking-type".into()); +static ANNOTATION_CHUNKING_TYPE: Lazy = + Lazy::new(|| crate::annotations::ANNOTATION_CHUNKING_TYPE.into()); + +/// Changes the type of the resolved module (only "json" is supported currently) +static ATTRIBUTE_MODULE_TYPE: Lazy = Lazy::new(|| "type".into()); impl ImportAnnotations { - fn insert(&mut self, key: JsWord, value: Option) { - self.map.insert(key, value); - } + pub fn parse(with: Option<&ObjectLit>) -> ImportAnnotations { + let Some(with) = with else { + return ImportAnnotations::default(); + }; + + let mut map = BTreeMap::new(); + + // The `with` clause is way more restrictive than `ObjectLit`, it only allows + // string -> value and value can only be a string. + // We just ignore everything else here till the SWC ast is more restrictive. + for (key, value) in with.props.iter().filter_map(|prop| { + let kv = prop.as_prop()?.as_key_value()?; - fn clear(&mut self) { - self.map.clear(); + let Lit::Str(str) = kv.value.as_lit()? else { + return None; + }; + + Some((&kv.key, str)) + }) { + let key = match key { + PropName::Ident(ident) => ident.sym.as_str(), + PropName::Str(str) => str.value.as_str(), + // the rest are invalid, ignore for now till SWC ast is correct + _ => continue, + }; + + map.insert(key.into(), value.value.as_str().into()); + } + + ImportAnnotations { map } } /// Returns the content on the transition annotation pub fn transition(&self) -> Option<&str> { - self.map - .get(&ANNOTATION_TRANSITION) - .and_then(|w| w.as_ref().map(|w| &**w)) + self.get(&ANNOTATION_TRANSITION) } /// Returns the content on the chunking-type annotation pub fn chunking_type(&self) -> Option<&str> { - self.map - .get(&ANNOTATION_CHUNKING_TYPE) - .and_then(|w| w.as_ref().map(|w| &**w)) + self.get(&ANNOTATION_CHUNKING_TYPE) + } + + /// Returns the content on the type attribute + pub fn module_type(&self) -> Option<&str> { + self.get(&ATTRIBUTE_MODULE_TYPE) + } + + pub fn get(&self, key: &JsWord) -> Option<&str> { + self.map.get(key).map(|w| w.as_str()) } } @@ -58,20 +91,12 @@ impl Display for ImportAnnotations { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut it = self.map.iter(); if let Some((k, v)) = it.next() { - if let Some(v) = v { - write!(f, "{{ {k}: {v}")? - } else { - write!(f, "{{ {k}")? - } + write!(f, "{{ {k}: {v}")? } else { return f.write_str("{}"); }; for (k, v) in it { - if let Some(v) = v { - write!(f, "; {k}: {v}")? - } else { - write!(f, "; {k}")? - } + write!(f, ", {k}: {v}")? } f.write_str(" }") } @@ -172,7 +197,6 @@ impl ImportMap { m.visit_with(&mut Analyzer { data: &mut data, - current_annotations: ImportAnnotations::default(), source, }); @@ -182,7 +206,6 @@ impl ImportMap { struct Analyzer<'a> { data: &'a mut ImportMap, - current_annotations: ImportAnnotations, source: Option>>, } @@ -222,42 +245,9 @@ fn to_word(name: &ModuleExportName) -> JsWord { } impl Visit for Analyzer<'_> { - fn visit_module_item(&mut self, n: &ModuleItem) { - if let ModuleItem::Stmt(Stmt::Expr(ExprStmt { expr, .. })) = n { - if let Expr::Lit(Lit::Str(s)) = unparen(expr) { - if s.value.starts_with("TURBOPACK") { - let value = &*s.value; - let value = value["TURBOPACK".len()..].trim(); - if !value.starts_with('{') || !value.ends_with('}') { - // TODO report issue - } else { - value[1..value.len() - 1] - .trim() - .split(';') - .map(|p| p.trim()) - .filter(|p| !p.is_empty()) - .for_each(|part| { - if let Some(colon) = part.find(':') { - self.current_annotations.insert( - part[..colon].trim_end().into(), - Some(part[colon + 1..].trim_start().into()), - ); - } else { - self.current_annotations.insert(part.into(), None); - } - }) - } - n.visit_children_with(self); - return; - } - } - } - n.visit_children_with(self); - self.current_annotations.clear(); - } - fn visit_import_decl(&mut self, import: &ImportDecl) { - let annotations = take(&mut self.current_annotations); + let annotations = ImportAnnotations::parse(import.with.as_deref()); + self.ensure_reference( import.span, import.src.value.clone(), @@ -295,7 +285,8 @@ impl Visit for Analyzer<'_> { fn visit_export_all(&mut self, export: &ExportAll) { self.data.has_exports = true; - let annotations = take(&mut self.current_annotations); + let annotations = ImportAnnotations::parse(export.with.as_deref()); + self.ensure_reference( export.span, export.src.value.clone(), @@ -313,53 +304,52 @@ impl Visit for Analyzer<'_> { fn visit_named_export(&mut self, export: &NamedExport) { self.data.has_exports = true; - if let Some(ref src) = export.src { - let annotations = take(&mut self.current_annotations); - self.ensure_reference( - export.span, - src.value.clone(), - ImportedSymbol::ModuleEvaluation, - annotations.clone(), - ); + let Some(ref src) = export.src else { + return; + }; - for spec in export.specifiers.iter() { - let symbol = get_import_symbol_from_export(spec); - - let i = self.ensure_reference( - export.span, - src.value.clone(), - symbol, - annotations.clone(), - ); - - match spec { - ExportSpecifier::Namespace(n) => { - self.data.reexports.push(( - i, - Reexport::Namespace { - exported: to_word(&n.name), - }, - )); - } - ExportSpecifier::Default(d) => { - self.data.reexports.push(( - i, - Reexport::Named { - imported: js_word!("default"), - exported: d.exported.sym.clone(), - }, - )); - } - ExportSpecifier::Named(n) => { - self.data.reexports.push(( - i, - Reexport::Named { - imported: to_word(&n.orig), - exported: to_word(n.exported.as_ref().unwrap_or(&n.orig)), - }, - )); - } + let annotations = ImportAnnotations::parse(export.with.as_deref()); + + self.ensure_reference( + export.span, + src.value.clone(), + ImportedSymbol::ModuleEvaluation, + annotations.clone(), + ); + + for spec in export.specifiers.iter() { + let symbol = get_import_symbol_from_export(spec); + + let i = + self.ensure_reference(export.span, src.value.clone(), symbol, annotations.clone()); + + match spec { + ExportSpecifier::Namespace(n) => { + self.data.reexports.push(( + i, + Reexport::Namespace { + exported: to_word(&n.name), + }, + )); + } + ExportSpecifier::Default(d) => { + self.data.reexports.push(( + i, + Reexport::Named { + imported: js_word!("default"), + exported: d.exported.sym.clone(), + }, + )); + } + ExportSpecifier::Named(n) => { + self.data.reexports.push(( + i, + Reexport::Named { + imported: to_word(&n.orig), + exported: to_word(n.exported.as_ref().unwrap_or(&n.orig)), + }, + )); } } } diff --git a/crates/turbopack-ecmascript/src/annotations.rs b/crates/turbopack-ecmascript/src/annotations.rs new file mode 100644 index 0000000000000..d6c2ba6847ea5 --- /dev/null +++ b/crates/turbopack-ecmascript/src/annotations.rs @@ -0,0 +1,34 @@ +use swc_core::{ + common::DUMMY_SP, + ecma::ast::{Expr, KeyValueProp, ObjectLit, Prop, PropName, PropOrSpread}, +}; + +/// Changes the chunking type for the annotated import +pub const ANNOTATION_CHUNKING_TYPE: &str = "turbopack-chunking-type"; + +/// Enables a specified transition for the annotated import +pub const ANNOTATION_TRANSITION: &str = "turbopack-transition"; + +pub fn with_chunking_type(chunking_type: &str) -> Box { + with_clause(&[(ANNOTATION_CHUNKING_TYPE, chunking_type)]) +} + +pub fn with_transition(transition_name: &str) -> Box { + with_clause(&[(ANNOTATION_TRANSITION, transition_name)]) +} + +pub fn with_clause<'a>( + entries: impl IntoIterator, +) -> Box { + Box::new(ObjectLit { + span: DUMMY_SP, + props: entries.into_iter().map(|(k, v)| with_prop(k, v)).collect(), + }) +} + +fn with_prop(key: &str, value: &str) -> PropOrSpread { + PropOrSpread::Prop(Box::new(Prop::KeyValue(KeyValueProp { + key: PropName::Str(key.into()), + value: Box::new(Expr::Lit(value.into())), + }))) +} diff --git a/crates/turbopack-ecmascript/src/lib.rs b/crates/turbopack-ecmascript/src/lib.rs index 089f02cdb49f8..1c3ae077d7522 100644 --- a/crates/turbopack-ecmascript/src/lib.rs +++ b/crates/turbopack-ecmascript/src/lib.rs @@ -8,6 +8,7 @@ #![recursion_limit = "256"] pub mod analyzer; +pub mod annotations; pub mod async_chunk; pub mod chunk; pub mod chunk_group_files_asset; diff --git a/crates/turbopack-ecmascript/src/references/esm/base.rs b/crates/turbopack-ecmascript/src/references/esm/base.rs index e829ca7154f9a..2488398c46052 100644 --- a/crates/turbopack-ecmascript/src/references/esm/base.rs +++ b/crates/turbopack-ecmascript/src/references/esm/base.rs @@ -14,7 +14,7 @@ use turbopack_core::{ issue::{IssueSeverity, IssueSource}, module::Module, reference::ModuleReference, - reference_type::EcmaScriptModulesReferenceSubType, + reference_type::{EcmaScriptModulesReferenceSubType, ImportWithType}, resolve::{ origin::{ResolveOrigin, ResolveOriginExt}, parse::Request, @@ -142,15 +142,18 @@ impl EsmAssetReference { impl ModuleReference for EsmAssetReference { #[turbo_tasks::function] async fn resolve_reference(&self) -> Result> { - let ty = Value::new(match &self.export_name { - Some(part) => EcmaScriptModulesReferenceSubType::ImportPart(*part), - None => EcmaScriptModulesReferenceSubType::Import, - }); + let ty = if matches!(self.annotations.module_type(), Some("json")) { + EcmaScriptModulesReferenceSubType::ImportWithType(ImportWithType::Json) + } else if let Some(part) = &self.export_name { + EcmaScriptModulesReferenceSubType::ImportPart(*part) + } else { + EcmaScriptModulesReferenceSubType::Import + }; Ok(esm_resolve( self.get_origin().resolve().await?, self.request, - ty, + Value::new(ty), IssueSeverity::Error.cell(), self.issue_source, )) @@ -162,7 +165,7 @@ impl ValueToString for EsmAssetReference { #[turbo_tasks::function] async fn to_string(&self) -> Result> { Ok(Vc::cell(format!( - "import {} {}", + "import {} with {}", self.request.to_string().await?, self.annotations ))) diff --git a/crates/turbopack-ecmascript/src/references/mod.rs b/crates/turbopack-ecmascript/src/references/mod.rs index 9d392edce8502..e651e0f7f00c9 100644 --- a/crates/turbopack-ecmascript/src/references/mod.rs +++ b/crates/turbopack-ecmascript/src/references/mod.rs @@ -29,10 +29,12 @@ use constant_value::ConstantValue; use indexmap::IndexSet; use lazy_static::lazy_static; use num_traits::Zero; +use once_cell::sync::Lazy; use parking_lot::Mutex; use regex::Regex; use sourcemap::decode_data_url; use swc_core::{ + atoms::JsWord, common::{ comments::{CommentKind, Comments}, errors::{DiagnosticId, Handler, HANDLER}, @@ -113,7 +115,7 @@ use crate::{ analyzer::{ builtin::early_replace_builtin, graph::{ConditionalKind, EffectArg, EvalContext, VarGraph}, - imports::{ImportedSymbol, Reexport}, + imports::{ImportAnnotations, ImportedSymbol, Reexport}, parse_require_context, top_level_await::has_top_level_await, ConstantNumber, ConstantString, ModuleValue, RequireContextValue, @@ -2792,18 +2794,12 @@ async fn resolve_as_webpack_runtime( #[turbo_tasks::value(transparent, serialization = "none")] pub struct AstPath(#[turbo_tasks(trace_ignore)] Vec); -pub static TURBOPACK_HELPER: &str = "__turbopackHelper"; +pub static TURBOPACK_HELPER: Lazy = Lazy::new(|| "__turbopack-helper__".into()); pub fn is_turbopack_helper_import(import: &ImportDecl) -> bool { - import.with.as_ref().map_or(false, |asserts| { - asserts.props.iter().any(|assert| { - assert - .as_prop() - .and_then(|prop| prop.as_key_value()) - .and_then(|kv| kv.key.as_ident()) - .map_or(false, |ident| &*ident.sym == TURBOPACK_HELPER) - }) - }) + let annotations = ImportAnnotations::parse(import.with.as_deref()); + + annotations.get(&TURBOPACK_HELPER).is_some() } #[derive(Debug)] diff --git a/crates/turbopack-resolve/src/ecmascript.rs b/crates/turbopack-resolve/src/ecmascript.rs index 9a51aeb567521..ef08b659de601 100644 --- a/crates/turbopack-resolve/src/ecmascript.rs +++ b/crates/turbopack-resolve/src/ecmascript.rs @@ -41,7 +41,10 @@ pub fn get_condition_maps( } #[turbo_tasks::function] -pub async fn apply_esm_specific_options(options: Vc) -> Result> { +pub async fn apply_esm_specific_options( + options: Vc, + reference_type: Value, +) -> Result> { let mut options: ResolveOptions = options.await?.clone_value(); // TODO set fully_specified when in strict ESM mode // options.fully_specified = true; @@ -49,6 +52,14 @@ pub async fn apply_esm_specific_options(options: Vc) -> Result>, ) -> Result> { let ty = Value::new(ReferenceType::EcmaScriptModules(ty.into_value())); - let options = apply_esm_specific_options(origin.resolve_options(ty.clone())) + let options = apply_esm_specific_options(origin.resolve_options(ty.clone()), ty.clone()) .resolve() .await?; specific_resolve(origin, request, options, ty, issue_severity, issue_source).await diff --git a/crates/turbopack/src/lib.rs b/crates/turbopack/src/lib.rs index 628389197b6de..480213a10bf53 100644 --- a/crates/turbopack/src/lib.rs +++ b/crates/turbopack/src/lib.rs @@ -43,7 +43,8 @@ use turbopack_core::{ output::OutputAsset, raw_module::RawModule, reference_type::{ - CssReferenceSubType, EcmaScriptModulesReferenceSubType, InnerAssets, ReferenceType, + CssReferenceSubType, EcmaScriptModulesReferenceSubType, ImportWithType, InnerAssets, + ReferenceType, }, resolve::{ options::ResolveOptions, origin::PlainResolveOrigin, parse::Request, resolve, ModulePart, @@ -473,9 +474,25 @@ async fn process_default_internal( ReferenceType::Internal(inner_assets) => Some(*inner_assets), _ => None, }; + + let mut has_type_attribute = false; + let mut current_source = source; - let mut current_module_type = None; + let mut current_module_type = match &reference_type { + ReferenceType::EcmaScriptModules(EcmaScriptModulesReferenceSubType::ImportWithType(ty)) => { + has_type_attribute = true; + + match ty { + ImportWithType::Json => Some(ModuleType::Json), + } + } + _ => None, + }; + for (i, rule) in options.await?.rules.iter().enumerate() { + if has_type_attribute && current_module_type.is_some() { + continue; + } if processed_rules.contains(&i) { continue; }