Skip to content

Commit

Permalink
refactor(transformer_dts): create a Program for codegen (#3679)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Boshen
Boshen committed Jun 15, 2024
1 parent 2717a1a commit 4f16664
Show file tree
Hide file tree
Showing 6 changed files with 122 additions and 97 deletions.
5 changes: 2 additions & 3 deletions crates/oxc_codegen/examples/codegen.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@ fn main() -> std::io::Result<()> {
println!("Original:");
println!("{source_text}");

let options = CodegenOptions { enable_source_map: false, ..Default::default() };
let printed = Codegen::<false>::new("", &source_text, ret.trivias, options)
let options = CodegenOptions::default();
let printed = Codegen::<false>::new("", &source_text, ret.trivias.clone(), options)
.build(&ret.program)
.source_text;
println!("Printed:");
println!("{printed}");

let ret = Parser::new(&allocator, &printed, source_type).parse();
let minified = Codegen::<true>::new("", &source_text, ret.trivias, options)
.build(&ret.program)
.source_text;
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_codegen/src/gen.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ impl<'a, const MINIFY: bool> Gen<MINIFY> for UsingDeclaration<'a> {
impl<'a, const MINIFY: bool> Gen<MINIFY> for VariableDeclaration<'a> {
fn gen(&self, p: &mut Codegen<{ MINIFY }>, ctx: Context) {
p.add_source_mapping(self.span.start);
if self.modifiers.contains(ModifierKind::Declare) {
if self.modifiers.is_contains_declare() {
p.print_str(b"declare ");
}

Expand Down
9 changes: 4 additions & 5 deletions crates/oxc_codegen/src/gen_ts.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use crate::context::Context;
use crate::{Codegen, Gen, GenExpr};
#[allow(clippy::wildcard_imports)]
use oxc_ast::ast::*;
use oxc_syntax::precedence::Precedence;

use crate::{context::Context, Codegen, Gen, GenExpr};

impl<'a, const MINIFY: bool> Gen<MINIFY> for TSTypeParameterDeclaration<'a> {
fn gen(&self, p: &mut Codegen<{ MINIFY }>, ctx: Context) {
p.print_str(b"<");
Expand Down Expand Up @@ -194,13 +194,12 @@ impl<'a, const MINIFY: bool> Gen<MINIFY> for TSType<'a> {
}
Self::TSTemplateLiteralType(decl) => decl.gen(p, ctx),
Self::TSTypeLiteral(decl) => {
p.print_str(b"{");
p.print_block_start(decl.span.start);
for item in &decl.members {
item.gen(p, ctx);
p.print_semicolon();
}
p.print_soft_space();
p.print_str(b"}");
p.print_block_end(decl.span.end);
}
Self::TSTypeOperatorType(decl) => {
match decl.operator {
Expand Down
40 changes: 17 additions & 23 deletions crates/oxc_codegen/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,23 @@ pub struct CodegenReturn {
pub source_map: Option<oxc_sourcemap::SourceMap>,
}

impl From<CodegenReturn> for String {
fn from(val: CodegenReturn) -> Self {
val.source_text
}
}

impl<'a, const MINIFY: bool> From<Codegen<'a, MINIFY>> for String {
fn from(mut val: Codegen<'a, MINIFY>) -> Self {
val.into_source_text()
}
}
impl<'a, const MINIFY: bool> From<Codegen<'a, MINIFY>> for Cow<'a, str> {
fn from(mut val: Codegen<'a, MINIFY>) -> Self {
Cow::Owned(val.into_source_text())
}
}

pub struct Codegen<'a, const MINIFY: bool> {
options: CodegenOptions,

Expand Down Expand Up @@ -456,12 +473,6 @@ impl<'a, const MINIFY: bool> Codegen<'a, MINIFY> {
}
}
for stmt in statements {
if let Some(decl) = stmt.as_declaration() {
if decl.is_typescript_syntax() && !matches!(decl, Declaration::TSEnumDeclaration(_))
{
continue;
}
}
if print_semicolon_first {
self.print_semicolon_if_needed();
stmt.gen(self, ctx);
Expand Down Expand Up @@ -502,20 +513,3 @@ fn choose_quote(s: &str) -> char {
'\''
}
}

impl From<CodegenReturn> for String {
fn from(val: CodegenReturn) -> Self {
val.source_text
}
}

impl<'a, const MINIFY: bool> From<Codegen<'a, MINIFY>> for String {
fn from(mut val: Codegen<'a, MINIFY>) -> Self {
val.into_source_text()
}
}
impl<'a, const MINIFY: bool> From<Codegen<'a, MINIFY>> for Cow<'a, str> {
fn from(mut val: Codegen<'a, MINIFY>) -> Self {
Cow::Owned(val.into_source_text())
}
}
121 changes: 65 additions & 56 deletions crates/oxc_transformer_dts/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ use oxc_allocator::Allocator;
use oxc_ast::Trivias;
#[allow(clippy::wildcard_imports)]
use oxc_ast::{ast::*, Visit};
use oxc_codegen::{Codegen, CodegenOptions, Context, Gen};
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_diagnostics::OxcDiagnostic;
use oxc_span::SPAN;
use oxc_span::{SourceType, SPAN};
use scope::ScopeTree;

pub struct TransformerDtsReturn {
Expand All @@ -34,27 +34,19 @@ pub struct TransformerDtsReturn {

pub struct TransformerDts<'a> {
ctx: Ctx<'a>,
codegen: Codegen<'a, false>,
scope: ScopeTree<'a>,
}

impl<'a> TransformerDts<'a> {
#[allow(clippy::needless_pass_by_value)]
pub fn new(
allocator: &'a Allocator,
source_path: &Path,
source_text: &'a str,
trivias: Trivias,
_source_path: &Path,
_source_text: &'a str,
_trivias: Trivias,
) -> Self {
let codegen = Codegen::new(
&source_path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default(),
source_text,
trivias,
CodegenOptions::default(),
);

let ctx = Rc::new(TransformDtsCtx::new(allocator));

Self { ctx, codegen, scope: ScopeTree::new() }
Self { ctx, scope: ScopeTree::new() }
}

/// # Errors
Expand All @@ -71,16 +63,20 @@ impl<'a> TransformerDts<'a> {
)
});

if has_import_or_export {
self.transform_program(program);
let stmts = if has_import_or_export {
self.transform_program(program)
} else {
self.transform_program_without_module_declaration(program);
}
self.transform_program_without_module_declaration(program)
};

TransformerDtsReturn {
source_text: self.codegen.into_source_text(),
errors: self.ctx.take_errors(),
}
let source_type = SourceType::default().with_module(true).with_typescript_definition(true);
let directives = self.ctx.ast.new_vec();
let program = self.ctx.ast.program(SPAN, source_type, directives, None, stmts);
let source_text =
Codegen::<false>::new("", "", Trivias::default(), CodegenOptions::default())
.build(&program)
.source_text;
TransformerDtsReturn { source_text, errors: self.ctx.take_errors() }
}

pub fn modifiers_declare(&self) -> Modifiers<'a> {
Expand All @@ -91,19 +87,27 @@ impl<'a> TransformerDts<'a> {
}

impl<'a> TransformerDts<'a> {
pub fn transform_program_without_module_declaration(&mut self, program: &Program<'a>) {
program.body.iter().for_each(|stmt| {
pub fn transform_program_without_module_declaration(
&mut self,
program: &Program<'a>,
) -> oxc_allocator::Vec<'a, Statement<'a>> {
let mut new_ast_stmts = self.ctx.ast.new_vec::<Statement<'a>>();
for stmt in &program.body {
if let Some(decl) = stmt.as_declaration() {
if let Some(decl) = self.transform_declaration(decl, false) {
decl.gen(&mut self.codegen, Context::empty());
new_ast_stmts.push(Statement::from(decl));
} else {
decl.gen(&mut self.codegen, Context::empty());
new_ast_stmts.push(Statement::from(self.ctx.ast.copy(decl)));
}
}
});
}
new_ast_stmts
}

pub fn transform_program(&mut self, program: &Program<'a>) {
pub fn transform_program(
&mut self,
program: &Program<'a>,
) -> oxc_allocator::Vec<'a, Statement<'a>> {
let mut new_stmts = Vec::new();
let mut variables_declarations = VecDeque::new();
let mut variable_transformed_indexes = VecDeque::new();
Expand Down Expand Up @@ -226,10 +230,11 @@ impl<'a> TransformerDts<'a> {

// 6. Transform variable/using declarations, import statements, remove unused imports
// 7. Generate code
for (index, stmt) in new_stmts.iter().enumerate() {
let mut new_ast_stmts = self.ctx.ast.new_vec::<Statement<'a>>();
for (index, stmt) in new_stmts.drain(..).enumerate() {
match stmt {
_ if transformed_indexes.contains(&index) => {
stmt.gen(&mut self.codegen, Context::empty());
new_ast_stmts.push(stmt);
}
Statement::VariableDeclaration(decl) => {
let indexes =
Expand All @@ -238,17 +243,18 @@ impl<'a> TransformerDts<'a> {
variables_declarations.pop_front().unwrap_or_else(|| unreachable!());

if !indexes.is_empty() {
self.transform_variable_declaration_with_new_declarations(
decl,
self.ctx.ast.new_vec_from_iter(
declarations
.into_iter()
.enumerate()
.filter(|(i, _)| indexes.contains(i))
.map(|(_, decl)| decl),
),
)
.gen(&mut self.codegen, Context::empty());
let variables_declaration = self
.transform_variable_declaration_with_new_declarations(
&decl,
self.ctx.ast.new_vec_from_iter(
declarations
.into_iter()
.enumerate()
.filter(|(i, _)| indexes.contains(i))
.map(|(_, decl)| decl),
),
);
new_ast_stmts.push(Statement::VariableDeclaration(variables_declaration));
}
}
Statement::UsingDeclaration(decl) => {
Expand All @@ -258,29 +264,32 @@ impl<'a> TransformerDts<'a> {
variables_declarations.pop_front().unwrap_or_else(|| unreachable!());

if !indexes.is_empty() {
self.transform_using_declaration_with_new_declarations(
decl,
self.ctx.ast.new_vec_from_iter(
declarations
.into_iter()
.enumerate()
.filter(|(i, _)| indexes.contains(i))
.map(|(_, decl)| decl),
),
)
.gen(&mut self.codegen, Context::empty());
let variable_declaration = self
.transform_using_declaration_with_new_declarations(
&decl,
self.ctx.ast.new_vec_from_iter(
declarations
.into_iter()
.enumerate()
.filter(|(i, _)| indexes.contains(i))
.map(|(_, decl)| decl),
),
);
new_ast_stmts.push(Statement::VariableDeclaration(variable_declaration));
}
}
Statement::ImportDeclaration(decl) => {
// We must transform this in the end, because we need to know all references
if decl.specifiers.is_none() {
decl.gen(&mut self.codegen, Context::empty());
} else if let Some(decl) = self.transform_import_declaration(decl) {
decl.gen(&mut self.codegen, Context::empty());
new_ast_stmts.push(Statement::ImportDeclaration(decl));
} else if let Some(decl) = self.transform_import_declaration(&decl) {
new_ast_stmts.push(Statement::ImportDeclaration(self.ctx.ast.alloc(decl)));
}
}
_ => {}
}
}

new_ast_stmts
}
}
42 changes: 33 additions & 9 deletions tasks/coverage/src/typescript/transpile_runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,30 @@ impl TypeScriptTranspileCase {
let filename = change_extension(self.path.to_str().unwrap());
let path =
project_root().join(TESTS_ROOT).join("baselines/reference/transpile").join(filename);
let expected_text = fs::read_to_string(path).unwrap();

// remove the error diagnostics lines
let expected_text = {
let raw_expected_text = fs::read_to_string(path).unwrap();
let mut expected_text = String::new();
let mut ignore = false;
for line in raw_expected_text.split("\r\n") {
if let Some(remain) = line.strip_prefix("//// ") {
ignore = remain.starts_with("[Diagnostics reported]");
if ignore {
continue;
}
}
if !ignore {
expected_text.push_str(line);
expected_text.push_str("\r\n");
}
}
expected_text
};

// compare lines
let baseline_lines = baseline_text.lines().collect::<Vec<_>>();
let expected_lines = expected_text.lines().collect::<Vec<_>>();
let baseline_lines = baseline_text.lines().filter(|s| !s.is_empty()).collect::<Vec<_>>();
let expected_lines = expected_text.lines().filter(|s| !s.is_empty()).collect::<Vec<_>>();
if baseline_lines.len() != expected_lines.len() {
return TestResult::Mismatch(baseline_text, expected_text);
}
Expand Down Expand Up @@ -147,12 +167,16 @@ impl TypeScriptTranspileCase {
if !ret.source_text.ends_with('\n') {
baseline_text.push_str("\r\n");
}
if !ret.errors.is_empty() {
baseline_text.push_str("\r\n\r\n//// [Diagnostics reported]\r\n");
for error in &ret.errors {
baseline_text.push_str(&error.message.to_string());
}
}
// ignore the diagnostics for now
// if !ret.errors.is_empty() {
// baseline_text.push_str("\r\n\r\n//// [Diagnostics reported]\r\n");
// for error in &ret.errors {
// baseline_text.push_str(&error.message.to_string());
// }
// if !baseline_text.ends_with('\n') {
// baseline_text.push_str("\r\n");
// }
// }
}

baseline_text
Expand Down

0 comments on commit 4f16664

Please sign in to comment.