Skip to content

Commit 9e51882

Browse files
weswighamweswigham
authored
Numeric separators (#20324)
* Add support into octal and binary literals * Add hex support * And finally support all numeric literals and fix spelling * Update error message * Refactor error in scanner to take a position * Scan no separators in escape sequences, add escape sequence tests * More decimal tests from the spec presentation examples * Permissive scanning of excess separators * Remove unnecessary assignment * Make code easier to follow
1 parent 0c2d8d2 commit 9e51882

File tree

45 files changed

+2834
-22
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2834
-22
lines changed

‎src/compiler/diagnosticMessages.json

+4
Original file line numberDiff line numberDiff line change
@@ -3420,6 +3420,10 @@
34203420
"category": "Message",
34213421
"code": 6187
34223422
},
3423+
"Numeric separators are not allowed here.": {
3424+
"category": "Error",
3425+
"code": 6188
3426+
},
34233427
"Variable '{0}' implicitly has an '{1}' type.": {
34243428
"category": "Error",
34253429
"code": 7005

‎src/compiler/scanner.ts

+107-20
Original file line numberDiff line numberDiff line change
@@ -561,9 +561,9 @@ namespace ts {
561561
return false;
562562
}
563563

564-
function scanConflictMarkerTrivia(text: string, pos: number, error?: ErrorCallback) {
564+
function scanConflictMarkerTrivia(text: string, pos: number, error?: (diag: DiagnosticMessage, pos?: number, len?: number) => void) {
565565
if (error) {
566-
error(Diagnostics.Merge_conflict_marker_encountered, mergeConflictMarkerLength);
566+
error(Diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength);
567567
}
568568

569569
const ch = text.charCodeAt(pos);
@@ -852,34 +852,86 @@ namespace ts {
852852
scanRange,
853853
};
854854

855-
function error(message: DiagnosticMessage, length?: number): void {
855+
function error(message: DiagnosticMessage): void;
856+
function error(message: DiagnosticMessage, errPos: number, length: number): void;
857+
function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void {
856858
if (onError) {
859+
const oldPos = pos;
860+
pos = errPos;
857861
onError(message, length || 0);
862+
pos = oldPos;
858863
}
859864
}
860865

866+
function scanNumberFragment(): string {
867+
let start = pos;
868+
let allowSeparator = false;
869+
let result = "";
870+
while (true) {
871+
const ch = text.charCodeAt(pos);
872+
if (ch === CharacterCodes._) {
873+
tokenFlags |= TokenFlags.ContainsSeparator;
874+
if (allowSeparator) {
875+
allowSeparator = false;
876+
result += text.substring(start, pos);
877+
}
878+
else {
879+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
880+
}
881+
pos++;
882+
start = pos;
883+
continue;
884+
}
885+
if (isDigit(ch)) {
886+
allowSeparator = true;
887+
pos++;
888+
continue;
889+
}
890+
break;
891+
}
892+
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
893+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
894+
}
895+
return result + text.substring(start, pos);
896+
}
897+
861898
function scanNumber(): string {
862899
const start = pos;
863-
while (isDigit(text.charCodeAt(pos))) pos++;
900+
const mainFragment = scanNumberFragment();
901+
let decimalFragment: string;
902+
let scientificFragment: string;
864903
if (text.charCodeAt(pos) === CharacterCodes.dot) {
865904
pos++;
866-
while (isDigit(text.charCodeAt(pos))) pos++;
905+
decimalFragment = scanNumberFragment();
867906
}
868907
let end = pos;
869908
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
870909
pos++;
871910
tokenFlags |= TokenFlags.Scientific;
872911
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
873-
if (isDigit(text.charCodeAt(pos))) {
874-
pos++;
875-
while (isDigit(text.charCodeAt(pos))) pos++;
876-
end = pos;
912+
const preNumericPart = pos;
913+
const finalFragment = scanNumberFragment();
914+
if (!finalFragment) {
915+
error(Diagnostics.Digit_expected);
877916
}
878917
else {
879-
error(Diagnostics.Digit_expected);
918+
scientificFragment = text.substring(end, preNumericPart) + finalFragment;
919+
end = pos;
920+
}
921+
}
922+
if (tokenFlags & TokenFlags.ContainsSeparator) {
923+
let result = mainFragment;
924+
if (decimalFragment) {
925+
result += "." + decimalFragment;
880926
}
927+
if (scientificFragment) {
928+
result += scientificFragment;
929+
}
930+
return "" + +result;
931+
}
932+
else {
933+
return "" + +(text.substring(start, end)); // No need to use all the fragments; no _ removal needed
881934
}
882-
return "" + +(text.substring(start, end));
883935
}
884936

885937
function scanOctalDigits(): number {
@@ -894,23 +946,36 @@ namespace ts {
894946
* Scans the given number of hexadecimal digits in the text,
895947
* returning -1 if the given number is unavailable.
896948
*/
897-
function scanExactNumberOfHexDigits(count: number): number {
898-
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false);
949+
function scanExactNumberOfHexDigits(count: number, canHaveSeparators: boolean): number {
950+
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false, canHaveSeparators);
899951
}
900952

901953
/**
902954
* Scans as many hexadecimal digits as are available in the text,
903955
* returning -1 if the given number of digits was unavailable.
904956
*/
905-
function scanMinimumNumberOfHexDigits(count: number): number {
906-
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true);
957+
function scanMinimumNumberOfHexDigits(count: number, canHaveSeparators: boolean): number {
958+
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true, canHaveSeparators);
907959
}
908960

909-
function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean): number {
961+
function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean, canHaveSeparators: boolean): number {
910962
let digits = 0;
911963
let value = 0;
964+
let allowSeparator = false;
912965
while (digits < minCount || scanAsManyAsPossible) {
913966
const ch = text.charCodeAt(pos);
967+
if (canHaveSeparators && ch === CharacterCodes._) {
968+
tokenFlags |= TokenFlags.ContainsSeparator;
969+
if (allowSeparator) {
970+
allowSeparator = false;
971+
}
972+
else {
973+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
974+
}
975+
pos++;
976+
continue;
977+
}
978+
allowSeparator = canHaveSeparators;
914979
if (ch >= CharacterCodes._0 && ch <= CharacterCodes._9) {
915980
value = value * 16 + ch - CharacterCodes._0;
916981
}
@@ -929,6 +994,9 @@ namespace ts {
929994
if (digits < minCount) {
930995
value = -1;
931996
}
997+
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
998+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
999+
}
9321000
return value;
9331001
}
9341002

@@ -1097,7 +1165,7 @@ namespace ts {
10971165
}
10981166

10991167
function scanHexadecimalEscape(numDigits: number): string {
1100-
const escapedValue = scanExactNumberOfHexDigits(numDigits);
1168+
const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false);
11011169

11021170
if (escapedValue >= 0) {
11031171
return String.fromCharCode(escapedValue);
@@ -1109,7 +1177,7 @@ namespace ts {
11091177
}
11101178

11111179
function scanExtendedUnicodeEscape(): string {
1112-
const escapedValue = scanMinimumNumberOfHexDigits(1);
1180+
const escapedValue = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
11131181
let isInvalidExtendedEscape = false;
11141182

11151183
// Validate the value of the digit
@@ -1162,7 +1230,7 @@ namespace ts {
11621230
if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) {
11631231
const start = pos;
11641232
pos += 2;
1165-
const value = scanExactNumberOfHexDigits(4);
1233+
const value = scanExactNumberOfHexDigits(4, /*canHaveSeparators*/ false);
11661234
pos = start;
11671235
return value;
11681236
}
@@ -1218,8 +1286,22 @@ namespace ts {
12181286
// For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b.
12191287
// Similarly valid octalIntegerLiteral must have at least one octal digit following o or O.
12201288
let numberOfDigits = 0;
1289+
let separatorAllowed = false;
12211290
while (true) {
12221291
const ch = text.charCodeAt(pos);
1292+
// Numeric seperators are allowed anywhere within a numeric literal, except not at the beginning, or following another separator
1293+
if (ch === CharacterCodes._) {
1294+
tokenFlags |= TokenFlags.ContainsSeparator;
1295+
if (separatorAllowed) {
1296+
separatorAllowed = false;
1297+
}
1298+
else {
1299+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
1300+
}
1301+
pos++;
1302+
continue;
1303+
}
1304+
separatorAllowed = true;
12231305
const valueOfCh = ch - CharacterCodes._0;
12241306
if (!isDigit(ch) || valueOfCh >= base) {
12251307
break;
@@ -1232,6 +1314,11 @@ namespace ts {
12321314
if (numberOfDigits === 0) {
12331315
return -1;
12341316
}
1317+
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
1318+
// Literal ends with underscore - not allowed
1319+
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
1320+
return value;
1321+
}
12351322
return value;
12361323
}
12371324

@@ -1435,7 +1522,7 @@ namespace ts {
14351522
case CharacterCodes._0:
14361523
if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
14371524
pos += 2;
1438-
let value = scanMinimumNumberOfHexDigits(1);
1525+
let value = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ true);
14391526
if (value < 0) {
14401527
error(Diagnostics.Hexadecimal_digit_expected);
14411528
value = 0;

‎src/compiler/types.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1490,8 +1490,9 @@ namespace ts {
14901490
HexSpecifier = 1 << 6, // e.g. `0x00000000`
14911491
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
14921492
OctalSpecifier = 1 << 8, // e.g. `0o777`
1493+
ContainsSeparator = 1 << 9, // e.g. `0b1100_0101`
14931494
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
1494-
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
1495+
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier | ContainsSeparator
14951496
}
14961497

14971498
export interface NumericLiteral extends LiteralExpression {

‎src/compiler/utilities.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ namespace ts {
346346
export function getLiteralText(node: LiteralLikeNode, sourceFile: SourceFile) {
347347
// If we don't need to downlevel and we can reach the original source text using
348348
// the node's parent reference, then simply get the text as it was originally written.
349-
if (!nodeIsSynthesized(node) && node.parent) {
349+
if (!nodeIsSynthesized(node) && node.parent && !(isNumericLiteral(node) && node.numericLiteralFlags & TokenFlags.ContainsSeparator)) {
350350
return getSourceTextOfNodeFromSourceFile(sourceFile, node);
351351
}
352352

‎tests/baselines/reference/parser.numericSeparators.binary.js

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
//// [parser.numericSeparators.binary.ts]
2+
0b00_11;
3+
0B0_1;
4+
0b1100_0011;
5+
0B0_11_0101;
6+
7+
8+
//// [parser.numericSeparators.binary.js]
9+
3;
10+
1;
11+
195;
12+
53;

‎tests/baselines/reference/parser.numericSeparators.binary.symbols

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
=== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/parser.numericSeparators.binary.ts ===
2+
0b00_11;
3+
No type information for this code.0B0_1;
4+
No type information for this code.0b1100_0011;
5+
No type information for this code.0B0_11_0101;
6+
No type information for this code.
7+
No type information for this code.

‎tests/baselines/reference/parser.numericSeparators.binary.types

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
=== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/parser.numericSeparators.binary.ts ===
2+
0b00_11;
3+
>0b00_11 : 3
4+
5+
0B0_1;
6+
>0B0_1 : 1
7+
8+
0b1100_0011;
9+
>0b1100_0011 : 195
10+
11+
0B0_11_0101;
12+
>0B0_11_0101 : 53
13+

‎tests/baselines/reference/parser.numericSeparators.binaryNegative.errors.txt

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/1.ts(1,5): error TS6188: Numeric separators are not allowed here.
2+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/2.ts(1,3): error TS6188: Numeric separators are not allowed here.
3+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/3.ts(1,2): error TS6188: Numeric separators are not allowed here.
4+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/3.ts(1,3): error TS1005: ';' expected.
5+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/3.ts(1,3): error TS2304: Cannot find name 'B0101'.
6+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/4.ts(1,6): error TS6188: Numeric separators are not allowed here.
7+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/5.ts(1,13): error TS6188: Numeric separators are not allowed here.
8+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/6.ts(1,3): error TS6188: Numeric separators are not allowed here.
9+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/6.ts(1,4): error TS6188: Numeric separators are not allowed here.
10+
tests/cases/conformance/parser/ecmascriptnext/numericSeparators/6.ts(1,5): error TS6188: Numeric separators are not allowed here.
11+
12+
13+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/1.ts (1 errors) ====
14+
0b00_
15+
~
16+
!!! error TS6188: Numeric separators are not allowed here.
17+
18+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/2.ts (1 errors) ====
19+
0b_110
20+
~
21+
!!! error TS6188: Numeric separators are not allowed here.
22+
23+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/3.ts (3 errors) ====
24+
0_B0101
25+
~
26+
!!! error TS6188: Numeric separators are not allowed here.
27+
~~~~~
28+
!!! error TS1005: ';' expected.
29+
~~~~~
30+
!!! error TS2304: Cannot find name 'B0101'.
31+
32+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/4.ts (1 errors) ====
33+
0b01__11
34+
~
35+
!!! error TS6188: Numeric separators are not allowed here.
36+
37+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/5.ts (1 errors) ====
38+
0B0110_0110__
39+
~
40+
!!! error TS6188: Numeric separators are not allowed here.
41+
42+
==== tests/cases/conformance/parser/ecmascriptnext/numericSeparators/6.ts (3 errors) ====
43+
0b___0111010_0101_1
44+
~
45+
!!! error TS6188: Numeric separators are not allowed here.
46+
~
47+
!!! error TS6188: Numeric separators are not allowed here.
48+
~
49+
!!! error TS6188: Numeric separators are not allowed here.
50+

‎tests/baselines/reference/parser.numericSeparators.binaryNegative.js

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//// [tests/cases/conformance/parser/ecmascriptnext/numericSeparators/parser.numericSeparators.binaryNegative.ts] ////
2+
3+
//// [1.ts]
4+
0b00_
5+
6+
//// [2.ts]
7+
0b_110
8+
9+
//// [3.ts]
10+
0_B0101
11+
12+
//// [4.ts]
13+
0b01__11
14+
15+
//// [5.ts]
16+
0B0110_0110__
17+
18+
//// [6.ts]
19+
0b___0111010_0101_1
20+
21+
22+
//// [1.js]
23+
0;
24+
//// [2.js]
25+
6;
26+
//// [3.js]
27+
0;
28+
B0101;
29+
//// [4.js]
30+
7;
31+
//// [5.js]
32+
102;
33+
//// [6.js]
34+
1867;

0 commit comments

Comments
 (0)