Skip to content
This repository has been archived by the owner on Nov 5, 2021. It is now read-only.

Improve tokenization of regular expressions #46

Merged
merged 2 commits into from
Oct 1, 2018
Merged

Improve tokenization of regular expressions #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3e6c3ee
Allow the 's' flag in regular expressions
spahnke Oct 1, 2018
ce3a065
Tokenize regex flags as 'keyword.other' like VS Code does
spahnke Oct 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion src/javascript/javascript.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ testTokenization('javascript', [
]
}],

[{
[{
line: 'x = /foo/.test(\'\')',
tokens: [
{ startIndex: 0, type: 'identifier.js' },
Expand All @@ -446,6 +446,39 @@ testTokenization('javascript', [
]
}],

[{
line: '/foo/',
tokens: [
{ startIndex: 0, type: 'regexp.js' }
]
}],

[{
line: '/foo/g',
tokens: [
{ startIndex: 0, type: 'regexp.js' },
{ startIndex: 5, type: 'keyword.other.js' }
]
}],

[{
line: '/foo/gimsuy',
tokens: [
{ startIndex: 0, type: 'regexp.js' },
{ startIndex: 5, type: 'keyword.other.js' }
]
}],

[{
line: '/foo/q', // invalid flag
tokens: [
{ startIndex: 0, type: 'delimiter.js' },
{ startIndex: 1, type: 'identifier.js' },
{ startIndex: 4, type: 'delimiter.js' },
{ startIndex: 5, type: 'identifier.js' }
]
}],

[{
line: 'x = 1 + f(2 / 3, /foo/)',
tokens: [
Expand Down
33 changes: 33 additions & 0 deletions src/typescript/typescript.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,39 @@ testTokenization('typescript', [
]
}],

[{
line: '/foo/',
tokens: [
{ startIndex: 0, type: 'regexp.ts' }
]
}],

[{
line: '/foo/g',
tokens: [
{ startIndex: 0, type: 'regexp.ts' },
{ startIndex: 5, type: 'keyword.other.ts' }
]
}],

[{
line: '/foo/gimsuy',
tokens: [
{ startIndex: 0, type: 'regexp.ts' },
{ startIndex: 5, type: 'keyword.other.ts' }
]
}],

[{
line: '/foo/q', // invalid flag
tokens: [
{ startIndex: 0, type: 'delimiter.ts' },
{ startIndex: 1, type: 'identifier.ts' },
{ startIndex: 4, type: 'delimiter.ts' },
{ startIndex: 5, type: 'identifier.ts' }
]
}],

[{
line: 'x = 1 + f(2 / 3, /foo/)',
tokens: [
Expand Down
4 changes: 2 additions & 2 deletions src/typescript/typescript.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ export const language = {
{ include: '@whitespace' },

// regular expression: ensure it is terminated before beginning (otherwise it is an opeator)
[/\/(?=([^\\\/]|\\.)+\/([gimuy]*)(\s*)(\.|;|\/|,|\)|\]|\}|$))/, { token: 'regexp', bracket: '@open', next: '@regexp' }],
[/\/(?=([^\\\/]|\\.)+\/([gimsuy]*)(\s*)(\.|;|\/|,|\)|\]|\}|$))/, { token: 'regexp', bracket: '@open', next: '@regexp' }],

// delimiters and operators
[/[()\[\]]/, '@brackets'],
Expand Down Expand Up @@ -190,7 +190,7 @@ export const language = {
[/[^\\\/]/, 'regexp'],
[/@regexpesc/, 'regexp.escape'],
[/\\\./, 'regexp.invalid'],
['/', { token: 'regexp', bracket: '@close' }, '@pop'],
[/(\/)([gimsuy]*)/, [{ token: 'regexp', bracket: '@close', next: '@pop' }, 'keyword.other']],
],

regexrange: [
Expand Down