Added the ability to dump the token stream or ast in bin. (#278)

* Added the ability to dump the token stream or ast in bin.

The dump functionality works both for files and REPL.

With --dump-tokens or -t for short it dumps the token stream to stdout  and --dump-ast or -a for short to dump the ast to stdout.

The dumping of tokens and ast is mutually exclusive. and when dumping it wont run the code.

* Fixed some issues with rustfmt.

* Added serde serialization and deserialization to token and the ast.

* Added a dynamic multi-format dumping of token stream and ast in bin.

- Changed the --dump-tokens and --dump-ast to be an optional argument that optionally takes a value of format type ([--opt=[val]]).
- The default format for --dump-tokens and --dump-ast is Debug format which calls std::fmt::Debug.
- Added Json and JsonMinified format for both dumps,  use serde_json internally.
- It is easy to support other format types, such as Toml with toml-rs for example.

* Made serde an optional dependency.

- Serde serialization and deserialization can be switched on by using the feature flag "serde-ast".

* Changed the JSON dumping format.

- Now Json  dumping format prints the data in minefied JSON form by default.
- Removed JsonMinified.
- Added JsonPretty as a way to dump the data in pretty printed JSON format.

* Updated the docs.

README.md

@@ -86,12 +86,18 @@ see [CHANGELOG](./CHANGELOG.md)
 
 ```
 USAGE:
-    boa_cli [FILE]...
+    boa_cli [OPTIONS] [FILE]...
 
 FLAGS:
     -h, --help       Prints help information
     -V, --version    Prints version information
 
+OPTIONS:
+    -a, --dump-ast <FORMAT>       Dump the ast to stdout with the given format [possible values: Debug, Json,
+                                  JsonPretty]
+    -t, --dump-tokens <FORMAT>    Dump the token stream to stdout with the given format [possible values: Debug, Json,
+                                  JsonPretty]
+
 ARGS:
     <FILE>...    The JavaScript file(s) to be evaluated
 ```

boa/Cargo.toml

@@ -11,6 +11,7 @@ exclude = ["../.vscode/*", "../Dockerfile", "../Makefile", "../.editorConfig"]
 edition = "2018"
 
 [features]
+serde-ast = ["serde"]
 default = ["wasm-bindgen"]
 
 [dependencies]
@@ -22,6 +23,7 @@ regex = "1.3.4"
 
 # Optional Dependencies
 wasm-bindgen = { version = "0.2.58", optional = true }
+serde = { version = "1.0", features = ["derive"], optional = true }
 
 [dev-dependencies]
 criterion = "0.3.1"

boa/src/lib.rs

@@ -19,6 +19,9 @@ use crate::{
     syntax::{ast::expr::Expr, lexer::Lexer, parser::Parser},
 };
 
+#[cfg(feature = "serde-ast")]
+pub use serde_json;
+
 fn parser_expr(src: &str) -> Result<Expr, String> {
     let mut lexer = Lexer::new(src);
     lexer.lex().map_err(|e| format!("SyntaxError: {}", e))?;

boa/src/syntax/ast/constant.rs

@@ -1,6 +1,10 @@
 use gc_derive::{Finalize, Trace};
 use std::fmt::{Display, Formatter, Result};
 
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A Javascript Constant
 pub enum Const {

boa/src/syntax/ast/expr.rs

@@ -8,6 +8,10 @@ use std::{
     fmt::{Display, Formatter, Result},
 };
 
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Trace, Finalize, Debug, PartialEq)]
 pub struct Expr {
     /// The expression definition
@@ -27,6 +31,7 @@ impl Display for Expr {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A Javascript Expression
 pub enum ExprDef {

boa/src/syntax/ast/keyword.rs

@@ -4,6 +4,10 @@ use std::{
     str::FromStr,
 };
 
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Copy, PartialEq, Debug)]
 /// A Javascript Keyword
 /// As specificed by <https://www.ecma-international.org/ecma-262/#sec-keywords>

boa/src/syntax/ast/op.rs

@@ -1,6 +1,9 @@
 use gc_derive::{Finalize, Trace};
 use std::fmt::{Display, Formatter, Result};
 
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
 /// Represents an operator
 pub trait Operator {
     /// Get the associativity as a boolean that is true if it goes rightwards
@@ -13,6 +16,7 @@ pub trait Operator {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A numeric operation between 2 values
 pub enum NumOp {
@@ -47,6 +51,7 @@ impl Display for NumOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A unary operation on a single value
 pub enum UnaryOp {
@@ -88,6 +93,7 @@ impl Display for UnaryOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A bitwise operation between 2 values
 pub enum BitOp {
@@ -119,6 +125,7 @@ impl Display for BitOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A comparitive operation between 2 values
 pub enum CompOp {
@@ -159,6 +166,7 @@ impl Display for CompOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A logical operation between 2 boolean values
 pub enum LogOp {
@@ -181,6 +189,7 @@ impl Display for LogOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A binary operation between 2 values
 pub enum BinOp {
@@ -240,6 +249,7 @@ impl Display for BinOp {
     }
 }
 
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug, Trace, Finalize, PartialEq)]
 /// A binary operation between 2 values
 pub enum AssignOp {

boa/src/syntax/ast/pos.rs

@@ -1,3 +1,7 @@
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, Copy, PartialEq, Debug)]
 /// A position in the Javascript source code
 /// Stores both the column number and the line number

boa/src/syntax/ast/punc.rs

@@ -1,5 +1,9 @@
 use std::fmt::{Display, Error, Formatter};
 
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(PartialEq, Clone, Copy, Debug)]
 /// Punctuation
 pub enum Punctuator {

boa/src/syntax/ast/token.rs

@@ -1,9 +1,12 @@
 use crate::syntax::ast::{keyword::Keyword, pos::Position, punc::Punctuator};
 use std::fmt::{Debug, Display, Formatter, Result};
 
-#[derive(Clone, PartialEq)]
+#[cfg(feature = "serde-ast")]
+use serde::{Deserialize, Serialize};
+
 /// Represents a token
-#[derive(Debug)]
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Token {
     /// The token Data
     pub data: TokenData,
@@ -38,7 +41,7 @@ impl Debug for VecToken {
         write!(f, "{}", buffer)
     }
 }
-
+#[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, PartialEq, Debug)]
 /// Represents the type of Token
 pub enum TokenData {

boa_cli/Cargo.toml

@@ -11,5 +11,5 @@ exclude = ["../.vscode/*", "../Dockerfile", "../Makefile", "../.editorConfig"]
 edition = "2018"
 
 [dependencies]
-Boa = { path = "../boa", default-features = false }
+Boa = { path = "../boa", features = ["serde-ast"], default-features = false }
 structopt = "0.3.9"