Skip to content
This repository was archived by the owner on May 19, 2018. It is now read-only.

NumberLiteralSeparator: Stage 1 feature plugin. Closes gh-538 #541

Merged
merged 4 commits into from
May 26, 2017
Merged

NumberLiteralSeparator: Stage 1 feature plugin. Closes gh-538 #541

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e9598bd
NumberLiteralSeparator: Stage 1 feature plugin
rwaldron May 26, 2017
244cade
fix-up per review
rwaldron May 26, 2017
d4ae979
nit: forbiddenNumericLiteralSeparatorSibling -> forbiddenNumericLiter…
rwaldron May 26, 2017
465d80b
fix-up to change includes -> indexOf
rwaldron May 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ require("babylon").parse("code", {
- `functionBind`
- `functionSent`
- `dynamicImport`
- `numericSeparator` ([proposal](https://github.com/samuelgoto/proposal-numeric-separator))

### FAQ

Expand Down
36 changes: 35 additions & 1 deletion src/tokenizer/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,23 @@ import { SourceLocation } from "../util/location";
import { lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace } from "../util/whitespace";
import State from "./state";


// The following character codes are forbidden from being
// an immediate sibling of NumericLiteralSeparator _

const forbiddenNumericLiteralSeparatorSiblings = [
46, // .
66, // B
69, // E
79, // O
88, // X
95, // _ (multiple separators are not allowed)
98, // b
101, // e
111, // o
120, // x
];

// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
Expand Down Expand Up @@ -555,6 +572,23 @@ export default class Tokenizer extends LocationParser {
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
const code = this.input.charCodeAt(this.state.pos);
let val;

if (this.hasPlugin("numericSeparator")) {
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) {
if (forbiddenNumericLiteralSeparatorSiblings.includes(prev) ||
Copy link
Member

@DrewML DrewML May 26, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the PR, @rwaldron! I believe we need to switch to Array#indexOf here instead of Array#includes. We don't load any polyfills in Babylon, and the minimum supported node version is 4.0.0.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dang forgot about that 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, ok, no problem. I saw all the other modern JS and just assumed I could use it...whoops! Will fix now.

forbiddenNumericLiteralSeparatorSiblings.includes(next) ||
Number.isNaN(next)) {
this.raise(this.state.pos, "Invalid NumericLiteralSeparator");
Copy link
Member

@hzoo hzoo May 26, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I might actually create a different error message for multi _, if it's at the end or the beginning, or around invalid chars but we can totally do that later too.

Could be a good 2nd PR

}

// Ignore this _ character
++this.state.pos;
continue;
}
}

if (code >= 97) {
val = code - 97 + 10; // a
} else if (code >= 65) {
Expand Down Expand Up @@ -608,7 +642,7 @@ export default class Tokenizer extends LocationParser {

if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");

const str = this.input.slice(start, this.state.pos);
const str = this.input.slice(start, this.state.pos).replace(/_/g, "");
let val;
if (isFloat) {
val = parseFloat(str);
Expand Down
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-0/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-0/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:1)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-1/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-1/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-10/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-10/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-11/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0xa_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-11/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-12/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x_a_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-12/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-13/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x__1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-13/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-14/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x_1__1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-14/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-15/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x_1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-15/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-16/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o_1_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-16/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-17/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o_11
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-17/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-18/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o_01_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-18/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-19/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0b_0_1_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-19/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-2/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1__
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-2/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-20/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0b_01_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-20/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-21/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0b01_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-21/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-22/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-22/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-23/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o_1_1_
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-23/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-24/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
._1_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-24/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Unexpected token (1:0)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-25/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o01_8
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-25/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Unexpected token, expected ; (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-26/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0b2_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-26/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Expected number in radix 2 (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-27/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0xZ_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-27/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Expected number in radix 16 (1:2)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-3/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1__1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-3/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:1)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-4/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_.1_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-4/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-5/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1._1_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-5/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:4)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-6/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_e1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-6/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-7/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_E1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-7/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-8/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1e_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-8/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-9/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1E_1
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/invalid-9/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
3 changes: 3 additions & 0 deletions test/fixtures/experimental/numeric-literal-separator/options.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"plugins": ["numericSeparator"]
}
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/valid-0/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1
69 changes: 69 additions & 0 deletions test/fixtures/experimental/numeric-literal-separator/valid-0/expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"program": {
"type": "Program",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"extra": {
"rawValue": 11,
"raw": "1_1"
},
"value": 11
}
}
],
"directives": []
}
}
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/valid-1/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_1
69 changes: 69 additions & 0 deletions test/fixtures/experimental/numeric-literal-separator/valid-1/expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"program": {
"type": "Program",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"extra": {
"rawValue": 11.11,
"raw": "1_1.1_1"
},
"value": 11.11
}
}
],
"directives": []
}
}
1 change: 1 addition & 0 deletions test/fixtures/experimental/numeric-literal-separator/valid-10/actual.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0o1_1
Loading