diff --git a/.github/.generated_ast_watch_list.yml b/.github/.generated_ast_watch_list.yml index ae30f7c9855ec..3b9a61fb66e04 100644 --- a/.github/.generated_ast_watch_list.yml +++ b/.github/.generated_ast_watch_list.yml @@ -13,9 +13,9 @@ src: - 'crates/oxc_ast/src/generated/assert_layouts.rs' - 'crates/oxc_ast/src/generated/ast_kind.rs' - 'crates/oxc_ast/src/generated/ast_builder.rs' + - 'crates/oxc_ast/src/generated/visit.rs' + - 'crates/oxc_ast/src/generated/visit_mut.rs' - 'crates/oxc_ast/src/generated/derive_clone_in.rs' - 'crates/oxc_ast/src/generated/derive_get_span.rs' - 'crates/oxc_ast/src/generated/derive_get_span_mut.rs' - - 'crates/oxc_ast/src/generated/visit.rs' - - 'crates/oxc_ast/src/generated/visit_mut.rs' - 'tasks/ast_codegen/src/**' diff --git a/crates/oxc_ast/src/ast/mod.rs b/crates/oxc_ast/src/ast/mod.rs index c48e79531414b..a02fbf82d51de 100644 --- a/crates/oxc_ast/src/ast/mod.rs +++ b/crates/oxc_ast/src/ast/mod.rs @@ -175,11 +175,11 @@ //! //! If you are seeing compile-time errors in `src/ast/macros.rs`, this will be the cause. -mod js; -mod jsx; -mod literal; -mod macros; -mod ts; +pub(crate) mod js; +pub(crate) mod jsx; +pub(crate) mod literal; +pub(crate) mod macros; +pub(crate) mod ts; use macros::inherit_variants; diff --git a/crates/oxc_ast/src/generated/derive_clone_in.rs b/crates/oxc_ast/src/generated/derive_clone_in.rs index 408a52c6dba71..c76df853ebbba 100644 --- a/crates/oxc_ast/src/generated/derive_clone_in.rs +++ b/crates/oxc_ast/src/generated/derive_clone_in.rs @@ -1,12 +1,21 @@ // Auto-generated code, DO NOT EDIT DIRECTLY! -// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_clone_in.rs` +// To edit this generated file you have to edit `tasks/ast_tools/src/derives/clone_in.rs` #![allow(clippy::default_trait_access)] use oxc_allocator::{Allocator, CloneIn}; #[allow(clippy::wildcard_imports)] -use crate::ast::*; +use crate::ast::js::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::jsx::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::literal::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::ts::*; impl<'alloc> CloneIn<'alloc> for BooleanLiteral { type Cloned = BooleanLiteral; diff --git a/crates/oxc_ast/src/generated/derive_get_span.rs b/crates/oxc_ast/src/generated/derive_get_span.rs index 85c72e8f2231e..38cde1c2ec246 100644 --- a/crates/oxc_ast/src/generated/derive_get_span.rs +++ b/crates/oxc_ast/src/generated/derive_get_span.rs @@ -1,12 +1,21 @@ // Auto-generated code, DO NOT EDIT DIRECTLY! -// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs` +// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs` #![allow(clippy::match_same_arms)] -use oxc_span::GetSpan; +use oxc_span::{GetSpan, Span}; #[allow(clippy::wildcard_imports)] -use crate::ast::*; +use crate::ast::js::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::jsx::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::literal::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::ts::*; impl GetSpan for BooleanLiteral { #[inline] diff --git a/crates/oxc_ast/src/generated/derive_get_span_mut.rs b/crates/oxc_ast/src/generated/derive_get_span_mut.rs index 450dfb891e3e0..a3c4f418cc03c 100644 --- a/crates/oxc_ast/src/generated/derive_get_span_mut.rs +++ b/crates/oxc_ast/src/generated/derive_get_span_mut.rs @@ -1,12 +1,21 @@ // Auto-generated code, DO NOT EDIT DIRECTLY! -// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs` +// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs` #![allow(clippy::match_same_arms)] -use oxc_span::GetSpanMut; +use oxc_span::{GetSpanMut, Span}; #[allow(clippy::wildcard_imports)] -use crate::ast::*; +use crate::ast::js::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::jsx::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::literal::*; + +#[allow(clippy::wildcard_imports)] +use crate::ast::ts::*; impl GetSpanMut for BooleanLiteral { #[inline] diff --git a/tasks/ast_tools/src/codegen.rs b/tasks/ast_tools/src/codegen.rs index b3c0f8066bccb..4722b444855c9 100644 --- a/tasks/ast_tools/src/codegen.rs +++ b/tasks/ast_tools/src/codegen.rs @@ -1,12 +1,16 @@ use std::{cell::RefCell, collections::HashMap, path::PathBuf}; use itertools::Itertools; +use proc_macro2::TokenStream; use crate::{ + derives::{Derive, DeriveOutput}, + fmt::pretty_print, generators::{Generator, GeneratorOutput}, passes::Pass, rust_ast::{self, AstRef}, schema::{lower_ast_types, Schema, TypeDef}, + util::write_all_to, Result, TypeId, }; @@ -15,16 +19,51 @@ pub struct AstCodegen { files: Vec, passes: Vec>>, generators: Vec>>, + derives: Vec>>, } pub struct AstCodegenResult { pub schema: Schema, - pub outputs: Vec<(/* generator name */ &'static str, /* output */ GeneratorOutput)>, + pub outputs: Vec, +} + +pub struct SideEffect(/* path */ pub PathBuf, /* output */ pub Vec); + +impl SideEffect { + /// Apply the side-effect + pub fn apply(self) -> std::io::Result<()> { + let Self(path, data) = self; + let path = path.into_os_string(); + let path = path.to_str().unwrap(); + write_all_to(&data, path)?; + Ok(()) + } + + #[allow(clippy::unnecessary_wraps)] + pub fn path(&self) -> Option { + let Self(path, _) = self; + let path = path.to_string_lossy(); + Some(path.replace('\\', "/")) + } +} + +impl From<(PathBuf, TokenStream)> for SideEffect { + fn from((path, stream): (PathBuf, TokenStream)) -> Self { + let content = pretty_print(&stream); + Self(path, content.as_bytes().into()) + } +} + +impl From for SideEffect { + fn from(output: GeneratorOutput) -> Self { + Self::from((output.0, output.1)) + } } pub trait Runner { type Context; type Output; + #[allow(dead_code)] fn name(&self) -> &'static str; fn run(&mut self, ctx: &Self::Context) -> Result; } @@ -116,7 +155,7 @@ impl AstCodegen { } #[must_use] - pub fn gen(mut self, generator: G) -> Self + pub fn generate(mut self, generator: G) -> Self where G: Generator + Runner + 'static, { @@ -124,7 +163,16 @@ impl AstCodegen { self } - pub fn generate(self) -> Result { + #[must_use] + pub fn derive(mut self, derive: D) -> Self + where + D: Derive + Runner + 'static, + { + self.derives.push(Box::new(derive)); + self + } + + pub fn run(self) -> Result { let modules = self .files .into_iter() @@ -140,17 +188,42 @@ impl AstCodegen { _ = self .passes .into_iter() - .map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res))) + .map(|mut runner| runner.run(&ctx)) .collect::>>()?; ctx.into_late_ctx() }; + let derives = self + .derives + .into_iter() + .map(|mut runner| runner.run(&ctx)) + .map_ok(|output| output.0.into_iter().map(SideEffect::from)) + .flatten_ok(); + let outputs = self .generators .into_iter() - .map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res))) + .map(|mut runner| runner.run(&ctx)) + .map_ok(SideEffect::from) + .chain(derives) .collect::>>()?; Ok(AstCodegenResult { outputs, schema: ctx.schema }) } } + +/// Creates a generated file warning + required information for a generated file. +macro_rules! generated_header { + () => {{ + let file = file!().replace("\\", "/"); + // TODO add generation date, AST source hash, etc here. + let edit_comment = format!("@ To edit this generated file you have to edit `{file}`"); + quote::quote! { + //!@ Auto-generated code, DO NOT EDIT DIRECTLY! + #![doc = #edit_comment] + //!@@line_break + } + }}; +} + +pub(crate) use generated_header; diff --git a/tasks/ast_tools/src/generators/derive_clone_in.rs b/tasks/ast_tools/src/derives/clone_in.rs similarity index 69% rename from tasks/ast_tools/src/generators/derive_clone_in.rs rename to tasks/ast_tools/src/derives/clone_in.rs index 5b4097766d0c7..6099b50ad82fb 100644 --- a/tasks/ast_tools/src/generators/derive_clone_in.rs +++ b/tasks/ast_tools/src/derives/clone_in.rs @@ -6,49 +6,34 @@ use syn::Ident; use crate::{ codegen::LateCtx, markers::CloneInAttribute, - output, schema::{EnumDef, GetIdent, StructDef, TypeDef}, - GeneratorOutput, }; -use super::{define_generator, generated_header, Generator}; +use super::{define_derive, Derive, DeriveOutput}; -define_generator! { +define_derive! { pub struct DeriveCloneIn; } -impl Generator for DeriveCloneIn { - fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput { - let impls: Vec = ctx - .schema() - .into_iter() - .filter(|def| def.generates_derive("CloneIn")) - .map(|def| match &def { - TypeDef::Enum(it) => derive_enum(it), - TypeDef::Struct(it) => derive_struct(it), - }) - .collect(); - - let header = generated_header!(); - - GeneratorOutput::Stream(( - output(crate::AST_CRATE, "derive_clone_in.rs"), - quote! { - #header - - #![allow(clippy::default_trait_access)] +impl Derive for DeriveCloneIn { + fn trait_name() -> &'static str { + "CloneIn" + } - ///@@line_break - use oxc_allocator::{Allocator, CloneIn}; + fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream { + match &def { + TypeDef::Enum(it) => derive_enum(it), + TypeDef::Struct(it) => derive_struct(it), + } + } - ///@@line_break - #[allow(clippy::wildcard_imports)] - use crate::ast::*; + fn prelude() -> TokenStream { + quote! { + #![allow(clippy::default_trait_access)] - ///@@line_break - #(#impls)* - }, - )) + ///@@line_break + use oxc_allocator::{Allocator, CloneIn}; + } } } @@ -106,7 +91,6 @@ fn impl_clone_in( ) -> TokenStream { if has_lifetime { quote! { - ///@@line_break impl <'old_alloc, 'new_alloc> CloneIn<'new_alloc> for #ty_ident<'old_alloc> { type Cloned = #ty_ident<'new_alloc>; fn clone_in(&self, #alloc_ident: &'new_alloc Allocator) -> Self::Cloned { @@ -116,7 +100,6 @@ fn impl_clone_in( } } else { quote! { - ///@@line_break impl <'alloc> CloneIn<'alloc> for #ty_ident { type Cloned = #ty_ident; fn clone_in(&self, #alloc_ident: &'alloc Allocator) -> Self::Cloned { diff --git a/tasks/ast_tools/src/generators/derive_get_span.rs b/tasks/ast_tools/src/derives/get_span.rs similarity index 58% rename from tasks/ast_tools/src/generators/derive_get_span.rs rename to tasks/ast_tools/src/derives/get_span.rs index bd84085361fa5..2eb0d9840d1d2 100644 --- a/tasks/ast_tools/src/generators/derive_get_span.rs +++ b/tasks/ast_tools/src/derives/get_span.rs @@ -4,41 +4,63 @@ use syn::Ident; use crate::{ codegen::LateCtx, - output, schema::{EnumDef, GetGenerics, StructDef, ToType, TypeDef}, util::ToIdent, - Generator, GeneratorOutput, }; -use super::{define_generator, generated_header}; +use super::{define_derive, Derive, DeriveOutput}; -define_generator! { +define_derive! { pub struct DeriveGetSpan; } -impl Generator for DeriveGetSpan { - fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput { +impl Derive for DeriveGetSpan { + fn trait_name() -> &'static str { + "GetSpan" + } + + fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream { let self_type = quote!(&self); let result_type = quote!(Span); let result_expr = quote!(self.span); - let out = derive("GetSpan", "span", &self_type, &result_type, &result_expr, ctx); - GeneratorOutput::Stream((output(crate::AST_CRATE, "derive_get_span.rs"), out)) + derive(Self::trait_name(), "span", &self_type, &result_type, &result_expr, def) + } + + fn prelude() -> TokenStream { + quote! { + #![allow(clippy::match_same_arms)] + + ///@@line_break + use oxc_span::{Span, GetSpan}; + } } } -define_generator! { +define_derive! { pub struct DeriveGetSpanMut; } -impl Generator for DeriveGetSpanMut { - fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput { +impl Derive for DeriveGetSpanMut { + fn trait_name() -> &'static str { + "GetSpanMut" + } + + fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream { let self_type = quote!(&mut self); let result_type = quote!(&mut Span); let result_expr = quote!(&mut self.span); - let out = derive("GetSpanMut", "span_mut", &self_type, &result_type, &result_expr, ctx); - GeneratorOutput::Stream((output(crate::AST_CRATE, "derive_get_span_mut.rs"), out)) + derive(Self::trait_name(), "span_mut", &self_type, &result_type, &result_expr, def) + } + + fn prelude() -> TokenStream { + quote! { + #![allow(clippy::match_same_arms)] + + ///@@line_break + use oxc_span::{Span, GetSpanMut}; + } } } @@ -48,40 +70,15 @@ fn derive( self_type: &TokenStream, result_type: &TokenStream, result_expr: &TokenStream, - ctx: &LateCtx, + def: &TypeDef, ) -> TokenStream { let trait_ident = trait_name.to_ident(); let method_ident = method_name.to_ident(); - let impls: Vec = ctx - .schema() - .into_iter() - .filter(|def| def.generates_derive(trait_name)) - .map(|def| match &def { - TypeDef::Enum(def) => { - derive_enum(def, &trait_ident, &method_ident, self_type, result_type) - } - TypeDef::Struct(def) => { - derive_struct(def, &trait_ident, &method_ident, self_type, result_type, result_expr) - } - }) - .collect(); - - let header = generated_header!(); - - quote! { - #header - - #![allow(clippy::match_same_arms)] - - ///@@line_break - use oxc_span::#trait_ident; - - ///@@line_break - #[allow(clippy::wildcard_imports)] - use crate::ast::*; - - ///@@line_break - #(#impls)* + match &def { + TypeDef::Enum(def) => derive_enum(def, &trait_ident, &method_ident, self_type, result_type), + TypeDef::Struct(def) => { + derive_struct(def, &trait_ident, &method_ident, self_type, result_type, result_expr) + } } } @@ -101,7 +98,6 @@ fn derive_enum( }); quote! { - ///@@line_break impl #generics #trait_name for #target_type { fn #method_name(#self_type) -> #result_type { match self { @@ -132,7 +128,6 @@ fn derive_struct( }; quote! { - ///@@line_break impl #generics #trait_name for #target_type { #[inline] fn #method_name(#self_type) -> #result_type { diff --git a/tasks/ast_tools/src/derives/mod.rs b/tasks/ast_tools/src/derives/mod.rs new file mode 100644 index 0000000000000..dbcf4c76ba5ac --- /dev/null +++ b/tasks/ast_tools/src/derives/mod.rs @@ -0,0 +1,129 @@ +use proc_macro2::TokenStream; +use std::path::PathBuf; + +use crate::{codegen::LateCtx, schema::TypeDef}; + +mod clone_in; +mod get_span; + +pub use clone_in::DeriveCloneIn; +pub use get_span::{DeriveGetSpan, DeriveGetSpanMut}; + +pub trait Derive { + fn trait_name() -> &'static str; + fn derive(&mut self, def: &TypeDef, ctx: &LateCtx) -> TokenStream; + fn prelude() -> TokenStream { + TokenStream::default() + } +} + +pub trait DeriveTemplate: Derive { + fn template(module_path: Vec<&str>, impls: TokenStream) -> TokenStream; +} + +#[derive(Debug, Clone)] +pub struct DeriveOutput(pub Vec<(PathBuf, TokenStream)>); + +macro_rules! define_derive { + ($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => { + $vis struct $ident $($lifetime)? $($rest)* + + impl $($lifetime)? $crate::derives::DeriveTemplate for $ident $($lifetime)? { + fn template(module_paths: Vec<&str>, impls: TokenStream) -> TokenStream { + use itertools::Itertools; + let header = $crate::codegen::generated_header!(); + let prelude = Self::prelude(); + + // from `x::y::z` to `crate::y::z::*` + let use_modules = module_paths.into_iter().map(|it|{ + let local_path = ["crate"] + .into_iter() + .chain(it.split("::").skip(1)) + .chain(["*"]) + .join("::"); + let use_module: syn::ItemUse = + syn::parse_str(format!("use {local_path};").as_str()).unwrap(); + quote::quote! { + ///@@line_break + #[allow(clippy::wildcard_imports)] + #use_module + } + }); + + quote::quote! { + #header + + #prelude + + #(#use_modules)* + + ///@@line_break + #impls + } + } + } + + impl $($lifetime)? $crate::codegen::Runner for $ident $($lifetime)? { + type Context = $crate::codegen::LateCtx; + type Output = $crate::derives::DeriveOutput; + + fn name(&self) -> &'static str { + stringify!($ident) + } + + fn run(&mut self, ctx: &$crate::codegen::LateCtx) -> $crate::Result { + use std::collections::{HashMap, HashSet}; + use std::vec::Vec; + use convert_case::{Case, Casing}; + use itertools::Itertools; + + use $crate::derives::DeriveTemplate; + + let trait_name = Self::trait_name(); + let output = ctx + .schema() + .into_iter() + .filter(|def| def.generates_derive(trait_name)) + .map(|def| (def, self.derive(def, ctx))) + .fold(HashMap::<&str, (HashSet<&str>, Vec)>::new(), |mut acc, (def, stream)| { + let module_path = def.module_path(); + let krate = module_path.split("::").next().unwrap(); + if !acc.contains_key(krate) { + acc.insert(krate, Default::default()); + } + let streams = acc.get_mut(krate).expect("We checked this right above!"); + streams.0.insert(module_path); + streams.1.push(stream); + acc + }) + .into_iter() + .sorted_by(|lhs, rhs| lhs.0.cmp(rhs.0)) + .fold(Vec::new(), |mut acc, (path, (modules, streams))| { + let mut modules = Vec::from_iter(modules); + modules.sort(); + acc.push(( + $crate::output( + format!("crates/{}", path.split("::").next().unwrap()).as_str(), + format!("derive_{}.rs", Self::trait_name().to_case(Case::Snake)).as_str() + ), + Self::template( + modules, + streams + .into_iter() + .fold(TokenStream::new(), |mut acc, it| { + acc.extend(quote::quote!{ + ///@@line_break + }); + acc.extend(it); + acc + }) + ) + )); + acc + }); + Ok(DeriveOutput(output)) + } + } + }; +} +pub(crate) use define_derive; diff --git a/tasks/ast_tools/src/generators/assert_layouts.rs b/tasks/ast_tools/src/generators/assert_layouts.rs index 49687117e93b8..b30f8189506f6 100644 --- a/tasks/ast_tools/src/generators/assert_layouts.rs +++ b/tasks/ast_tools/src/generators/assert_layouts.rs @@ -3,14 +3,14 @@ use quote::quote; use syn::Type; use crate::{ - codegen::LateCtx, + codegen::{generated_header, LateCtx}, output, schema::{FieldDef, ToType, TypeDef}, util::ToIdent, Generator, GeneratorOutput, }; -use super::{define_generator, generated_header}; +use super::define_generator; define_generator! { pub struct AssertLayouts; @@ -29,7 +29,7 @@ impl Generator for AssertLayouts { let header = generated_header!(); - GeneratorOutput::Stream(( + GeneratorOutput( output(crate::AST_CRATE, "assert_layouts.rs"), quote! { #header @@ -52,7 +52,7 @@ impl Generator for AssertLayouts { #[cfg(not(any(target_pointer_width = "64", target_pointer_width = "32")))] const _: () = panic!("Platforms with pointer width other than 64 or 32 bit are not supported"); }, - )) + ) } } diff --git a/tasks/ast_tools/src/generators/ast_builder.rs b/tasks/ast_tools/src/generators/ast_builder.rs index a7f6ef27afc8e..28366b1ec780c 100644 --- a/tasks/ast_tools/src/generators/ast_builder.rs +++ b/tasks/ast_tools/src/generators/ast_builder.rs @@ -8,9 +8,9 @@ use proc_macro2::TokenStream; use quote::{format_ident, quote, ToTokens}; use syn::{parse_quote, Ident, Type}; +use crate::codegen::generated_header; use crate::{ codegen::LateCtx, - generators::generated_header, output, schema::{ EnumDef, FieldDef, GetIdent, InheritDef, StructDef, ToType, TypeDef, TypeName, VariantDef, @@ -36,7 +36,7 @@ impl Generator for AstBuilderGenerator { let header = generated_header!(); - GeneratorOutput::Stream(( + GeneratorOutput( output(crate::AST_CRATE, "ast_builder.rs"), quote! { #header @@ -66,7 +66,7 @@ impl Generator for AstBuilderGenerator { #(#fns)* } }, - )) + ) } } diff --git a/tasks/ast_tools/src/generators/ast_kind.rs b/tasks/ast_tools/src/generators/ast_kind.rs index a43112c6fb2f3..874058e80f6af 100644 --- a/tasks/ast_tools/src/generators/ast_kind.rs +++ b/tasks/ast_tools/src/generators/ast_kind.rs @@ -3,14 +3,14 @@ use quote::quote; use syn::{parse_quote, Arm, Ident, Type, Variant}; use crate::{ - codegen::LateCtx, + codegen::{generated_header, LateCtx}, output, schema::{GetIdent, ToType, TypeDef}, util::ToIdent, Generator, GeneratorOutput, }; -use super::{define_generator, generated_header}; +use super::define_generator; define_generator! { pub struct AstKindGenerator; @@ -156,7 +156,7 @@ impl Generator for AstKindGenerator { let header = generated_header!(); - GeneratorOutput::Stream(( + GeneratorOutput( output(crate::AST_CRATE, "ast_kind.rs"), quote! { #header @@ -190,6 +190,6 @@ impl Generator for AstKindGenerator { } } }, - )) + ) } } diff --git a/tasks/ast_tools/src/generators/mod.rs b/tasks/ast_tools/src/generators/mod.rs index d4fb22db0e701..d7a02c1839028 100644 --- a/tasks/ast_tools/src/generators/mod.rs +++ b/tasks/ast_tools/src/generators/mod.rs @@ -7,15 +7,11 @@ use crate::codegen::LateCtx; mod assert_layouts; mod ast_builder; mod ast_kind; -mod derive_clone_in; -mod derive_get_span; mod visit; pub use assert_layouts::AssertLayouts; pub use ast_builder::AstBuilderGenerator; pub use ast_kind::AstKindGenerator; -pub use derive_clone_in::DeriveCloneIn; -pub use derive_get_span::{DeriveGetSpan, DeriveGetSpanMut}; pub use visit::{VisitGenerator, VisitMutGenerator}; /// Inserts a newline in the `TokenStream`. @@ -30,78 +26,8 @@ pub trait Generator { fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput; } -pub type GeneratedTokenStream = (/* output path */ PathBuf, TokenStream); -pub type GeneratedDataStream = (/* output path */ PathBuf, Vec); - -// TODO: remove me -#[allow(dead_code)] #[derive(Debug, Clone)] -pub enum GeneratorOutput { - None, - Info(Vec), - Data(GeneratedDataStream), - Stream(GeneratedTokenStream), -} - -// TODO: remove me -#[allow(dead_code)] -impl GeneratorOutput { - pub fn is_none(&self) -> bool { - matches!(self, Self::None) - } - - pub fn expect_none(&self) { - assert!(self.is_none()); - } - - pub fn to_info(&self) -> &[u8] { - if let Self::Info(it) = self { - it - } else { - panic!(); - } - } - - pub fn to_data(&self) -> &GeneratedDataStream { - if let Self::Data(it) = self { - it - } else { - panic!(); - } - } - - pub fn to_stream(&self) -> &GeneratedTokenStream { - if let Self::Stream(it) = self { - it - } else { - panic!(); - } - } - - pub fn into_info(self) -> Vec { - if let Self::Info(it) = self { - it - } else { - panic!(); - } - } - - pub fn into_data(self) -> GeneratedDataStream { - if let Self::Data(it) = self { - it - } else { - panic!(); - } - } - - pub fn into_stream(self) -> GeneratedTokenStream { - if let Self::Stream(it) = self { - it - } else { - panic!(); - } - } -} +pub struct GeneratorOutput(/* output path */ pub PathBuf, pub TokenStream); macro_rules! define_generator { ($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => { @@ -134,19 +60,3 @@ macro_rules! insert { } #[allow(unused_imports)] pub(crate) use insert; - -/// Creates a generated file warning + required information for a generated file. -macro_rules! generated_header { - () => {{ - let file = file!().replace("\\", "/"); - // TODO add generation date, AST source hash, etc here. - let edit_comment = format!("@ To edit this generated file you have to edit `{file}`"); - quote::quote! { - //!@ Auto-generated code, DO NOT EDIT DIRECTLY! - #![doc = #edit_comment] - //!@@line_break - } - }}; -} - -pub(crate) use generated_header; diff --git a/tasks/ast_tools/src/generators/visit.rs b/tasks/ast_tools/src/generators/visit.rs index 8a0cd1fad4d61..f57070bb41564 100644 --- a/tasks/ast_tools/src/generators/visit.rs +++ b/tasks/ast_tools/src/generators/visit.rs @@ -7,7 +7,7 @@ use quote::{format_ident, quote, ToTokens}; use syn::{parse_quote, Ident}; use crate::{ - codegen::LateCtx, + codegen::{generated_header, LateCtx}, generators::ast_kind::BLACK_LIST as KIND_BLACK_LIST, markers::VisitArg, output, @@ -16,7 +16,7 @@ use crate::{ Generator, GeneratorOutput, }; -use super::{define_generator, generated_header}; +use super::define_generator; define_generator! { pub struct VisitGenerator; @@ -28,19 +28,13 @@ define_generator! { impl Generator for VisitGenerator { fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput { - GeneratorOutput::Stream(( - output(crate::AST_CRATE, "visit.rs"), - generate_visit::(ctx), - )) + GeneratorOutput(output(crate::AST_CRATE, "visit.rs"), generate_visit::(ctx)) } } impl Generator for VisitMutGenerator { fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput { - GeneratorOutput::Stream(( - output(crate::AST_CRATE, "visit_mut.rs"), - generate_visit::(ctx), - )) + GeneratorOutput(output(crate::AST_CRATE, "visit_mut.rs"), generate_visit::(ctx)) } } diff --git a/tasks/ast_tools/src/main.rs b/tasks/ast_tools/src/main.rs index de0c9aa857e35..5f4e885e4a113 100644 --- a/tasks/ast_tools/src/main.rs +++ b/tasks/ast_tools/src/main.rs @@ -6,6 +6,7 @@ use itertools::Itertools; use syn::parse_file; mod codegen; +mod derives; mod fmt; mod generators; mod layout; @@ -15,10 +16,10 @@ mod rust_ast; mod schema; mod util; -use fmt::{cargo_fmt, pretty_print}; +use derives::{DeriveCloneIn, DeriveGetSpan, DeriveGetSpanMut}; +use fmt::cargo_fmt; use generators::{ - AssertLayouts, AstBuilderGenerator, AstKindGenerator, DeriveCloneIn, DeriveGetSpan, - DeriveGetSpanMut, GeneratedDataStream, GeneratedTokenStream, Generator, GeneratorOutput, + AssertLayouts, AstBuilderGenerator, AstKindGenerator, Generator, GeneratorOutput, VisitGenerator, VisitMutGenerator, }; use passes::{CalcLayout, Linker}; @@ -60,28 +61,25 @@ fn main() -> std::result::Result<(), Box> { .fold(AstCodegen::default(), AstCodegen::add_file) .pass(Linker) .pass(CalcLayout) - .gen(AssertLayouts) - .gen(AstKindGenerator) - .gen(AstBuilderGenerator) - .gen(DeriveCloneIn) - .gen(DeriveGetSpan) - .gen(DeriveGetSpanMut) - .gen(VisitGenerator) - .gen(VisitMutGenerator) - .generate()?; - - let (streams, outputs): (Vec<_>, Vec<_>) = - outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Stream(_))); - - let (binaries, _): (Vec<_>, Vec<_>) = - outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Data(_))); + .derive(DeriveCloneIn) + .derive(DeriveGetSpan) + .derive(DeriveGetSpanMut) + .generate(AssertLayouts) + .generate(AstKindGenerator) + .generate(AstBuilderGenerator) + .generate(VisitGenerator) + .generate(VisitMutGenerator) + .run()?; if !cli_options.dry_run { - let side_effects = - write_generated_streams(streams.into_iter().map(|it| it.1.into_stream()))? - .into_iter() - .chain(write_data_streams(binaries.into_iter().map(|it| it.1.into_data()))?) - .collect(); + let side_effects = outputs + .into_iter() + .filter_map(|it| { + let path = it.path(); + it.apply().unwrap(); + path + }) + .collect(); write_ci_filter(SOURCE_PATHS, side_effects, ".github/.generated_ast_watch_list.yml")?; } @@ -102,37 +100,6 @@ fn output(krate: &str, path: &str) -> PathBuf { std::path::PathBuf::from_iter(vec![krate, "src", "generated", path]) } -/// Writes all streams and returns a vector pointing to side-effects written on the disk -fn write_generated_streams( - streams: impl IntoIterator, -) -> std::io::Result> { - streams - .into_iter() - .map(|(path, stream)| { - let path = path.into_os_string(); - let path = path.to_str().unwrap(); - let content = pretty_print(&stream); - write_all_to(content.as_bytes(), path)?; - Ok(path.to_string().replace('\\', "/")) - }) - .collect() -} - -/// Writes all streams and returns a vector pointing to side-effects written on the disk -fn write_data_streams( - streams: impl IntoIterator, -) -> std::io::Result> { - streams - .into_iter() - .map(|(path, stream)| { - let path = path.into_os_string(); - let path = path.to_str().unwrap(); - write_all_to(&stream, path)?; - Ok(path.to_string().replace('\\', "/")) - }) - .collect() -} - fn write_ci_filter( inputs: &[&str], side_effects: Vec, diff --git a/tasks/ast_tools/src/rust_ast.rs b/tasks/ast_tools/src/rust_ast.rs index b95a16b172be5..64200dc674fd4 100644 --- a/tasks/ast_tools/src/rust_ast.rs +++ b/tasks/ast_tools/src/rust_ast.rs @@ -30,13 +30,27 @@ impl From for Inherit { } } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct EnumMeta { pub inherits: Vec, pub layout_32: Layout, pub layout_64: Layout, pub visitable: bool, pub ast: bool, + pub module_path: String, +} + +impl EnumMeta { + fn new(module_path: String) -> Self { + Self { + inherits: Vec::default(), + layout_32: Layout::default(), + layout_64: Layout::default(), + visitable: false, + ast: false, + module_path, + } + } } #[derive(Debug)] @@ -61,19 +75,26 @@ impl Enum { } } -impl From for Enum { - fn from(item: ItemEnum) -> Self { - Self { item, meta: EnumMeta::default() } - } -} - /// Placeholder for now! -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct StructMeta { pub layout_32: Layout, pub layout_64: Layout, pub visitable: bool, pub ast: bool, + pub module_path: String, +} + +impl StructMeta { + fn new(module_path: String) -> Self { + Self { + layout_32: Layout::default(), + layout_64: Layout::default(), + visitable: false, + ast: false, + module_path, + } + } } #[derive(Debug)] @@ -83,6 +104,10 @@ pub struct Struct { } impl Struct { + pub fn with_meta(item: ItemStruct, meta: StructMeta) -> Self { + Self { item, meta } + } + pub fn ident(&self) -> &Ident { &self.item.ident } @@ -94,9 +119,26 @@ impl Struct { } } -impl From for Struct { - fn from(item: ItemStruct) -> Self { - Self { item, meta: StructMeta::default() } +#[derive(Debug)] +pub struct Macro { + pub item: ItemMacro, + pub meta: MacroMeta, +} + +impl Macro { + pub fn with_meta(item: ItemMacro, meta: MacroMeta) -> Self { + Self { item, meta } + } +} + +#[derive(Debug)] +pub struct MacroMeta { + pub module_path: String, +} + +impl MacroMeta { + fn new(module_path: String) -> Self { + Self { module_path } } } @@ -106,7 +148,7 @@ pub enum AstType { Struct(Struct), // we need this to expand `inherit` macro calls. - Macro(ItemMacro), + Macro(Macro), } impl ToTokens for AstType { @@ -114,18 +156,30 @@ impl ToTokens for AstType { match self { Self::Enum(it) => it.item.to_tokens(tokens), Self::Struct(it) => it.item.to_tokens(tokens), - - Self::Macro(it) => it.to_tokens(tokens), + Self::Macro(it) => it.item.to_tokens(tokens), } } } impl AstType { + fn new(item: Item, module_path: String) -> Result { + match item { + Item::Enum(it) => Ok(AstType::Enum(Enum::with_meta(it, EnumMeta::new(module_path)))), + Item::Struct(it) => { + Ok(AstType::Struct(Struct::with_meta(it, StructMeta::new(module_path)))) + } + Item::Macro(it) => { + Ok(AstType::Macro(Macro::with_meta(it, MacroMeta::new(module_path)))) + } + _ => Err(String::from("Unsupported Item!")), + } + } + pub fn ident(&self) -> Option<&Ident> { match self { AstType::Enum(ty) => Some(ty.ident()), AstType::Struct(ty) => Some(ty.ident()), - AstType::Macro(tt) => tt.ident.as_ref(), + AstType::Macro(ty) => ty.item.ident.as_ref(), } } @@ -156,7 +210,7 @@ impl AstType { match self { AstType::Enum(it) => assign!(it), AstType::Struct(it) => assign!(it), - AstType::Macro(it) => return Err(unexpanded_macro_err(it)), + AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)), } Ok(()) } @@ -165,7 +219,7 @@ impl AstType { match self { AstType::Enum(it) => it.meta.ast = value, AstType::Struct(it) => it.meta.ast = value, - AstType::Macro(it) => return Err(unexpanded_macro_err(it)), + AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)), } Ok(()) } @@ -174,7 +228,7 @@ impl AstType { match self { AstType::Enum(it) => Ok(it.meta.layout_32.clone()), AstType::Struct(it) => Ok(it.meta.layout_32.clone()), - AstType::Macro(it) => Err(unexpanded_macro_err(it)), + AstType::Macro(it) => Err(unexpanded_macro_err(&it.item)), } } @@ -182,7 +236,7 @@ impl AstType { match self { AstType::Enum(it) => Ok(it.meta.layout_64.clone()), AstType::Struct(it) => Ok(it.meta.layout_64.clone()), - AstType::Macro(it) => Err(unexpanded_macro_err(it)), + AstType::Macro(it) => Err(unexpanded_macro_err(&it.item)), } } @@ -200,20 +254,16 @@ impl AstType { match self { AstType::Enum(it) => assign!(it), AstType::Struct(it) => assign!(it), - AstType::Macro(it) => return Err(unexpanded_macro_err(it)), + AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)), } Ok(()) } -} -impl TryFrom for AstType { - type Error = String; - fn try_from(item: Item) -> Result { - match item { - Item::Enum(it) => Ok(AstType::Enum(it.into())), - Item::Struct(it) => Ok(AstType::Struct(it.into())), - Item::Macro(it) => Ok(AstType::Macro(it)), - _ => Err(String::from("Unsupported Item!")), + pub fn module_path(&self) -> String { + match self { + AstType::Enum(it) => it.meta.module_path.clone(), + AstType::Struct(it) => it.meta.module_path.clone(), + AstType::Macro(it) => it.meta.module_path.clone(), } } } @@ -221,11 +271,8 @@ impl TryFrom for AstType { const LOAD_ERROR: &str = "should be loaded by now!"; #[derive(Debug)] pub struct Module { - pub path: PathBuf, - // TODO: remove me - #[allow(dead_code)] - #[allow(clippy::struct_field_names)] - pub module: String, + pub file: PathBuf, + pub path: String, pub shebang: Option, pub attrs: Vec, pub items: Vec, @@ -240,15 +287,25 @@ impl ToTokens for Module { } impl Module { - pub fn with_path(path: PathBuf) -> Self { - let module = path.file_stem().map(|it| it.to_string_lossy().to_string()).unwrap(); - Self { path, module, shebang: None, attrs: Vec::new(), items: Vec::new(), loaded: false } + /// Expects a file path to a rust source file in the `crates` directory. + pub fn with_path(file: PathBuf) -> Self { + let path = { + let no_ext = file.with_extension(""); + let string = no_ext.to_string_lossy(); + let mut parts = string.split('/'); + assert_eq!(parts.next(), Some("crates")); + let krate = parts.next().unwrap(); + assert_eq!(parts.next(), Some("src")); + let mut parts = [krate].into_iter().chain(parts); + parts.join("::") + }; + Self { file, path, shebang: None, attrs: Vec::new(), items: Vec::new(), loaded: false } } pub fn load(mut self) -> Result { assert!(!self.loaded, "can't load twice!"); - let mut file = std::fs::File::open(&self.path).normalize()?; + let mut file = std::fs::File::open(&self.file).normalize()?; let mut content = String::new(); file.read_to_string(&mut content).normalize()?; let file = parse_file(content.as_str()).normalize()?; @@ -263,7 +320,7 @@ impl Module { Item::Macro(m) if m.mac.path.is_ident("inherit_variants") => true, _ => false, }) - .map(TryInto::try_into) + .map(|it| AstType::new(it, self.path.clone())) .map_ok(|it| Rc::new(RefCell::new(it))) .collect::>()?; self.loaded = true; @@ -294,8 +351,9 @@ impl Module { pub fn expand(ast_ref: &AstRef) -> Result<()> { let to_replace = match &*ast_ref.borrow() { - AstType::Macro(mac) => { + ast_ref @ AstType::Macro(mac) => { let (enum_, inherits) = mac + .item .mac .parse_body_with(|input: &ParseBuffer| { // Because of `@inherit`s we can't use the actual `ItemEnum` parse, @@ -349,7 +407,7 @@ pub fn expand(ast_ref: &AstRef) -> Result<()> { enum_, EnumMeta { inherits: inherits.into_iter().map(Into::into).collect(), - ..EnumMeta::default() + ..EnumMeta::new(ast_ref.module_path()) }, ))) } diff --git a/tasks/ast_tools/src/schema/defs.rs b/tasks/ast_tools/src/schema/defs.rs index cfa9bc1695f9a..218c23867b2cb 100644 --- a/tasks/ast_tools/src/schema/defs.rs +++ b/tasks/ast_tools/src/schema/defs.rs @@ -36,6 +36,10 @@ impl TypeDef { let generated_derives = self.generated_derives(); generated_derives.iter().any(|it| it == derive) } + + pub fn module_path(&self) -> &str { + with_either!(self, it => &it.module_path) + } } #[derive(Debug, Serialize)] @@ -58,6 +62,8 @@ pub struct StructDef { pub generated_derives: Vec, #[serde(skip)] pub markers: OuterMarkers, + #[serde(skip)] + pub module_path: String, } #[derive(Debug, Serialize)] @@ -77,6 +83,8 @@ pub struct EnumDef { pub align_32: usize, pub offsets_32: Option>, pub generated_derives: Vec, + #[serde(skip)] + pub module_path: String, } impl EnumDef { diff --git a/tasks/ast_tools/src/schema/mod.rs b/tasks/ast_tools/src/schema/mod.rs index bb8791fb4c2b3..3c81d93c0609e 100644 --- a/tasks/ast_tools/src/schema/mod.rs +++ b/tasks/ast_tools/src/schema/mod.rs @@ -131,7 +131,7 @@ fn lower_ast_type(ty: &rust::AstType, ctx: &codegen::EarlyCtx) -> TypeDef { match ty { rust::AstType::Enum(it) => TypeDef::Enum(lower_ast_enum(it, ctx)), rust::AstType::Struct(it) => TypeDef::Struct(lower_ast_struct(it, ctx)), - rust::AstType::Macro(it) => panic!("{}", unexpanded_macro_err(it)), + rust::AstType::Macro(it) => panic!("{}", unexpanded_macro_err(&it.item)), } } @@ -167,6 +167,8 @@ fn lower_ast_enum(it @ rust::Enum { item, meta }: &rust::Enum, ctx: &codegen::Ea offsets_32, generated_derives: parse_generate_derive(&item.attrs), + + module_path: meta.module_path.clone(), } } @@ -190,15 +192,18 @@ fn lower_ast_struct( visitable: meta.visitable, fields: item.fields.iter().map(|fi| lower_field(fi, ctx)).collect(), has_lifetime: item.generics.lifetimes().count() > 0, + size_64, align_64, offsets_64, size_32, align_32, offsets_32, - markers: parse_outer_markers(&item.attrs).unwrap(), + markers: parse_outer_markers(&item.attrs).unwrap(), generated_derives: parse_generate_derive(&item.attrs), + + module_path: meta.module_path.clone(), } }