Skip to content

Commit 86028ad

Browse files
RunDevelopmentRunDevelopment
authoredNov 22, 2021
Ruby: Improved tokenization (#3193)
·
v1.30.0v1.26.0
1 parent 6add768 commit 86028ad

23 files changed

+1109
-525
lines changed
 

‎components/prism-crystal.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/
1212
});
1313

14-
Prism.languages.insertBefore('crystal', 'string', {
14+
Prism.languages.insertBefore('crystal', 'string-literal', {
1515
attribute: {
1616
pattern: /@\[.+?\]/,
1717
alias: 'attr-name',

‎components/prism-crystal.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎components/prism-ruby.js

+134-83
Original file line numberDiff line numberDiff line change
@@ -6,133 +6,184 @@
66
*/
77
(function (Prism) {
88
Prism.languages.ruby = Prism.languages.extend('clike', {
9-
'comment': [
10-
/#.*/,
11-
{
12-
pattern: /^=begin\s[\s\S]*?^=end/m,
13-
greedy: true
14-
}
15-
],
9+
'comment': {
10+
pattern: /#.*|^=begin\s[\s\S]*?^=end/m,
11+
greedy: true
12+
},
1613
'class-name': {
17-
pattern: /(\b(?:class)\s+|\bcatch\s+\()[\w.\\]+/i,
14+
pattern: /(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/,
1815
lookbehind: true,
1916
inside: {
2017
'punctuation': /[.\\]/
2118
}
2219
},
23-
'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/
20+
'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/,
21+
'operator': /\.{2,3}|&\.|===|<?=>|[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/,
22+
'punctuation': /[(){}[\].,;]/,
23+
});
24+
25+
Prism.languages.insertBefore('ruby', 'operator', {
26+
'double-colon': {
27+
pattern: /::/,
28+
alias: 'punctuation'
29+
},
2430
});
2531

2632
var interpolation = {
27-
pattern: /#\{[^}]+\}/,
33+
pattern: /((?:^|[^\\])(?:\\{2})*)#\{(?:[^{}]|\{[^{}]*\})*\}/,
34+
lookbehind: true,
2835
inside: {
36+
'content': {
37+
pattern: /^(#\{)[\s\S]+(?=\}$)/,
38+
lookbehind: true,
39+
inside: Prism.languages.ruby
40+
},
2941
'delimiter': {
3042
pattern: /^#\{|\}$/,
31-
alias: 'tag'
32-
},
33-
rest: Prism.languages.ruby
43+
alias: 'punctuation'
44+
}
3445
}
3546
};
3647

3748
delete Prism.languages.ruby.function;
3849

50+
var percentExpression = '(?:' + [
51+
/([^a-zA-Z0-9\s{(\[<=])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
52+
/\((?:[^()\\]|\\[\s\S]|\((?:[^()\\]|\\[\s\S])*\))*\)/.source,
53+
/\{(?:[^{}\\]|\\[\s\S]|\{(?:[^{}\\]|\\[\s\S])*\})*\}/.source,
54+
/\[(?:[^\[\]\\]|\\[\s\S]|\[(?:[^\[\]\\]|\\[\s\S])*\])*\]/.source,
55+
/<(?:[^<>\\]|\\[\s\S]|<(?:[^<>\\]|\\[\s\S])*>)*>/.source
56+
].join('|') + ')';
57+
58+
var symbolName = /(?:"(?:\\.|[^"\\\r\n])*"|(?:\b[a-zA-Z_]\w*|[^\s\0-\x7F]+)[?!]?|\$.)/.source;
59+
3960
Prism.languages.insertBefore('ruby', 'keyword', {
40-
'regex': [
61+
'regex-literal': [
4162
{
42-
pattern: RegExp(/%r/.source + '(?:' + [
43-
/([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
44-
/\((?:[^()\\]|\\[\s\S])*\)/.source,
45-
// Here we need to specifically allow interpolation
46-
/\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source,
47-
/\[(?:[^\[\]\\]|\\[\s\S])*\]/.source,
48-
/<(?:[^<>\\]|\\[\s\S])*>/.source
49-
].join('|') + ')' + /[egimnosux]{0,6}/.source),
63+
pattern: RegExp(/%r/.source + percentExpression + /[egimnosux]{0,6}/.source),
5064
greedy: true,
5165
inside: {
52-
'interpolation': interpolation
66+
'interpolation': interpolation,
67+
'regex': /[\s\S]+/
5368
}
5469
},
5570
{
5671
pattern: /(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/,
5772
lookbehind: true,
5873
greedy: true,
5974
inside: {
60-
'interpolation': interpolation
75+
'interpolation': interpolation,
76+
'regex': /[\s\S]+/
6177
}
6278
}
6379
],
6480
'variable': /[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/,
65-
'symbol': {
66-
pattern: /(^|[^:]):[a-zA-Z_]\w*(?:[?!]|\b)/,
67-
lookbehind: true
68-
},
81+
'symbol': [
82+
{
83+
pattern: RegExp(/(^|[^:]):/.source + symbolName),
84+
lookbehind: true,
85+
greedy: true
86+
},
87+
{
88+
pattern: RegExp(/([\r\n{(,][ \t]*)/.source + symbolName + /(?=:(?!:))/.source),
89+
lookbehind: true,
90+
greedy: true
91+
},
92+
],
6993
'method-definition': {
70-
pattern: /(\bdef\s+)[\w.]+/,
94+
pattern: /(\bdef\s+)\w+(?:\s*\.\s*\w+)?/,
7195
lookbehind: true,
7296
inside: {
73-
'function': /\w+$/,
74-
rest: Prism.languages.ruby
97+
'function': /\b\w+$/,
98+
'keyword': /^self\b/,
99+
'class-name': /^\w+/,
100+
'punctuation': /\./
75101
}
76102
}
77103
});
78104

79-
Prism.languages.insertBefore('ruby', 'number', {
80-
'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,
81-
'constant': /\b[A-Z]\w*(?:[?!]|\b)/
82-
});
83-
84-
Prism.languages.ruby.string = [
85-
{
86-
pattern: RegExp(/%[qQiIwWxs]?/.source + '(?:' + [
87-
/([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
88-
/\((?:[^()\\]|\\[\s\S])*\)/.source,
89-
// Here we need to specifically allow interpolation
90-
/\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source,
91-
/\[(?:[^\[\]\\]|\\[\s\S])*\]/.source,
92-
/<(?:[^<>\\]|\\[\s\S])*>/.source
93-
].join('|') + ')'),
94-
greedy: true,
95-
inside: {
96-
'interpolation': interpolation
97-
}
98-
},
99-
{
100-
pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,
101-
greedy: true,
102-
inside: {
103-
'interpolation': interpolation
105+
Prism.languages.insertBefore('ruby', 'string', {
106+
'string-literal': [
107+
{
108+
pattern: RegExp(/%[qQiIwWs]?/.source + percentExpression),
109+
greedy: true,
110+
inside: {
111+
'interpolation': interpolation,
112+
'string': /[\s\S]+/
113+
}
114+
},
115+
{
116+
pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,
117+
greedy: true,
118+
inside: {
119+
'interpolation': interpolation,
120+
'string': /[\s\S]+/
121+
}
122+
},
123+
{
124+
pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
125+
alias: 'heredoc-string',
126+
greedy: true,
127+
inside: {
128+
'delimiter': {
129+
pattern: /^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i,
130+
inside: {
131+
'symbol': /\b\w+/,
132+
'punctuation': /^<<[-~]?/
133+
}
134+
},
135+
'interpolation': interpolation,
136+
'string': /[\s\S]+/
137+
}
138+
},
139+
{
140+
pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
141+
alias: 'heredoc-string',
142+
greedy: true,
143+
inside: {
144+
'delimiter': {
145+
pattern: /^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i,
146+
inside: {
147+
'symbol': /\b\w+/,
148+
'punctuation': /^<<[-~]?'|'$/,
149+
}
150+
},
151+
'string': /[\s\S]+/
152+
}
104153
}
105-
},
106-
{
107-
pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
108-
alias: 'heredoc-string',
109-
greedy: true,
110-
inside: {
111-
'delimiter': {
112-
pattern: /^<<[-~]?[a-z_]\w*|[a-z_]\w*$/i,
113-
alias: 'symbol',
114-
inside: {
115-
'punctuation': /^<<[-~]?/
154+
],
155+
'command-literal': [
156+
{
157+
pattern: RegExp(/%x/.source + percentExpression),
158+
greedy: true,
159+
inside: {
160+
'interpolation': interpolation,
161+
'command': {
162+
pattern: /[\s\S]+/,
163+
alias: 'string'
116164
}
117-
},
118-
'interpolation': interpolation
119-
}
120-
},
121-
{
122-
pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
123-
alias: 'heredoc-string',
124-
greedy: true,
125-
inside: {
126-
'delimiter': {
127-
pattern: /^<<[-~]?'[a-z_]\w*'|[a-z_]\w*$/i,
128-
alias: 'symbol',
129-
inside: {
130-
'punctuation': /^<<[-~]?'|'$/,
165+
}
166+
},
167+
{
168+
pattern: /`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/,
169+
greedy: true,
170+
inside: {
171+
'interpolation': interpolation,
172+
'command': {
173+
pattern: /[\s\S]+/,
174+
alias: 'string'
131175
}
132176
}
133177
}
134-
}
135-
];
178+
]
179+
});
180+
181+
delete Prism.languages.ruby.string;
182+
183+
Prism.languages.insertBefore('ruby', 'number', {
184+
'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,
185+
'constant': /\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/
186+
});
136187

137188
Prism.languages.rb = Prism.languages.ruby;
138189
}(Prism));

‎components/prism-ruby.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/crystal/attribute_feature.test

+39-2
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,54 @@
11
@[AlwaysInline]
22
@[CallConvention("X86_StdCall")]
3+
@[MyAnnotation(key: "value", value: 123)]
4+
@[MyAnnotation("foo", 123, false)]
35

46
----------------------------------------------------
57

68
[
79
["attribute", [
810
["delimiter", "@["],
9-
["constant", "AlwaysInline"],
11+
"AlwaysInline",
1012
["delimiter", "]"]
1113
]],
1214
["attribute", [
1315
["delimiter", "@["],
14-
["constant", "CallConvention"], ["punctuation", "("], ["string", [ "\"X86_StdCall\"" ]], ["punctuation", ")"],
16+
"CallConvention",
17+
["punctuation", "("],
18+
["string-literal", [
19+
["string", "\"X86_StdCall\""]
20+
]],
21+
["punctuation", ")"],
22+
["delimiter", "]"]
23+
]],
24+
["attribute", [
25+
["delimiter", "@["],
26+
"MyAnnotation",
27+
["punctuation", "("],
28+
["symbol", "key"],
29+
["operator", ":"],
30+
["string-literal", [
31+
["string", "\"value\""]
32+
]],
33+
["punctuation", ","],
34+
["symbol", "value"],
35+
["operator", ":"],
36+
["number", "123"],
37+
["punctuation", ")"],
38+
["delimiter", "]"]
39+
]],
40+
["attribute", [
41+
["delimiter", "@["],
42+
"MyAnnotation",
43+
["punctuation", "("],
44+
["string-literal", [
45+
["string", "\"foo\""]
46+
]],
47+
["punctuation", ","],
48+
["number", "123"],
49+
["punctuation", ","],
50+
["boolean", "false"],
51+
["punctuation", ")"],
1552
["delimiter", "]"]
1653
]]
1754
]

‎tests/languages/crystal/keyword_feature.test

+108-24
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,119 @@
1-
abstract alias as asm begin break case
2-
class;
3-
def;
4-
do else elsif
5-
end ensure enum extend for fun
6-
if include instance_sizeof
71
.is_a?
8-
lib macro module next of out pointerof
9-
private protected rescue
102
.responds_to?
11-
return require select self sizeof struct super
12-
then type typeof uninitialized union unless
13-
until when while with yield __DIR__ __END_LINE__
14-
__FILE__ __LINE__
3+
abstract
4+
alias
5+
as
6+
asm
7+
begin
8+
break
9+
case
10+
class;
11+
def;
12+
do
13+
else
14+
elsif
15+
end
16+
ensure
17+
enum
18+
extend
19+
for
20+
fun
21+
if
22+
include
23+
instance_sizeof
24+
lib
25+
macro
26+
module;
27+
next
28+
of
29+
out
30+
pointerof
31+
private
32+
protected
33+
require
34+
rescue
35+
return
36+
select
37+
self
38+
sizeof
39+
struct
40+
super
41+
then
42+
type
43+
typeof
44+
uninitialized
45+
union
46+
unless
47+
until
48+
when
49+
while
50+
with
51+
yield
52+
53+
__DIR__
54+
__END_LINE__
55+
__FILE__
56+
__LINE__
1557

1658
----------------------------------------------------
1759

1860
[
19-
["keyword", "abstract"], ["keyword", "alias"], ["keyword", "as"], ["keyword", "asm"], ["keyword", "begin"], ["keyword", "break"], ["keyword", "case"],
20-
["keyword", "class"], ["punctuation", ";"],
21-
["keyword", "def"], ["punctuation", ";"],
22-
["keyword", "do"], ["keyword", "else"], ["keyword", "elsif"],
23-
["keyword", "end"], ["keyword", "ensure"], ["keyword", "enum"], ["keyword", "extend"], ["keyword", "for"], ["keyword", "fun"],
24-
["keyword", "if"], ["keyword", "include"], ["keyword", "instance_sizeof"],
2561
["punctuation", "."], ["keyword", "is_a?"],
26-
["keyword", "lib"], ["keyword", "macro"], ["keyword", "module"], ["keyword", "next"], ["keyword", "of"], ["keyword", "out"], ["keyword", "pointerof"],
27-
["keyword", "private"], ["keyword", "protected"], ["keyword", "rescue"],
2862
["punctuation", "."], ["keyword", "responds_to?"],
29-
["keyword", "return"], ["keyword", "require"], ["keyword", "select"], ["keyword", "self"], ["keyword", "sizeof"], ["keyword", "struct"], ["keyword", "super"],
30-
["keyword", "then"], ["keyword", "type"], ["keyword", "typeof"], ["keyword", "uninitialized"], ["keyword", "union"], ["keyword", "unless"],
31-
["keyword", "until"], ["keyword", "when"], ["keyword", "while"], ["keyword", "with"], ["keyword", "yield"], ["keyword", "__DIR__"], ["keyword", "__END_LINE__"],
32-
["keyword", "__FILE__"], ["keyword", "__LINE__"]
63+
["keyword", "abstract"],
64+
["keyword", "alias"],
65+
["keyword", "as"],
66+
["keyword", "asm"],
67+
["keyword", "begin"],
68+
["keyword", "break"],
69+
["keyword", "case"],
70+
["keyword", "class"], ["punctuation", ";"],
71+
["keyword", "def"], ["punctuation", ";"],
72+
["keyword", "do"],
73+
["keyword", "else"],
74+
["keyword", "elsif"],
75+
["keyword", "end"],
76+
["keyword", "ensure"],
77+
["keyword", "enum"],
78+
["keyword", "extend"],
79+
["keyword", "for"],
80+
["keyword", "fun"],
81+
["keyword", "if"],
82+
["keyword", "include"],
83+
["keyword", "instance_sizeof"],
84+
["keyword", "lib"],
85+
["keyword", "macro"],
86+
["keyword", "module"], ["punctuation", ";"],
87+
["keyword", "next"],
88+
["keyword", "of"],
89+
["keyword", "out"],
90+
["keyword", "pointerof"],
91+
["keyword", "private"],
92+
["keyword", "protected"],
93+
["keyword", "require"],
94+
["keyword", "rescue"],
95+
["keyword", "return"],
96+
["keyword", "select"],
97+
["keyword", "self"],
98+
["keyword", "sizeof"],
99+
["keyword", "struct"],
100+
["keyword", "super"],
101+
["keyword", "then"],
102+
["keyword", "type"],
103+
["keyword", "typeof"],
104+
["keyword", "uninitialized"],
105+
["keyword", "union"],
106+
["keyword", "unless"],
107+
["keyword", "until"],
108+
["keyword", "when"],
109+
["keyword", "while"],
110+
["keyword", "with"],
111+
["keyword", "yield"],
112+
113+
["keyword", "__DIR__"],
114+
["keyword", "__END_LINE__"],
115+
["keyword", "__FILE__"],
116+
["keyword", "__LINE__"]
33117
]
34118

35119
----------------------------------------------------

‎tests/languages/erb+haml/erb_inclusion.test

+6-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
" render ",
1717
["variable", "@products"],
1818
["operator", "||"],
19-
["string", ["\"empty_list\""]]
19+
["string-literal", [
20+
["string", "\"empty_list\""]
21+
]]
2022
]],
2123
["delimiter", "%>"]
2224
]]
@@ -30,7 +32,9 @@
3032
" render ",
3133
["variable", "@products"],
3234
["operator", "||"],
33-
["string", ["\"empty_list\""]]
35+
["string-literal", [
36+
["string", "\"empty_list\""]
37+
]]
3438
]],
3539
["delimiter", "%>"]
3640
]]

‎tests/languages/erb/erb_feature.test

+3-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
" render ",
1919
["variable", "@products"],
2020
["operator", "||"],
21-
["string", ["\"empty_list\""]]
21+
["string-literal", [
22+
["string", "\"empty_list\""]
23+
]]
2224
]],
2325
["delimiter", "%>"]
2426
]],

‎tests/languages/erb/erb_in_markup_feature.test

+3-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ ___ERB1___<%= 1 %>___ERB2___<%= 2 %>
2222
["punctuation", "."],
2323
"strftime",
2424
["punctuation", "("],
25-
["string", ["'%A'"]],
25+
["string-literal", [
26+
["string", "'%A'"]
27+
]],
2628
["punctuation", ")"]
2729
]],
2830
["delimiter", "%>"]

‎tests/languages/haml/interpolation_feature.test

+3-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
["interpolation", [
1515
["delimiter", "#{"],
1616
["ruby", [
17-
["string", ["\"foobar\""]]
17+
["string-literal", [
18+
["string", "\"foobar\""]
19+
]]
1820
]],
1921
["delimiter", "}"]
2022
]]

‎tests/languages/haml/tag_feature.test

+16-14
Original file line numberDiff line numberDiff line change
@@ -42,22 +42,24 @@
4242
["attributes", [
4343
["punctuation", "{"],
4444
["symbol", ":type"],
45-
["operator", "="],
46-
["operator", ">"],
47-
["string", ["\"text/javascript\""]],
45+
["operator", "=>"],
46+
["string-literal", [
47+
["string", "\"text/javascript\""]
48+
]],
4849
["punctuation", ","],
4950

5051
["symbol", ":src"],
51-
["operator", "="],
52-
["operator", ">"],
53-
["string", [
54-
"\"javascripts/script_",
52+
["operator", "=>"],
53+
["string-literal", [
54+
["string", "\"javascripts/script_"],
5555
["interpolation", [
5656
["delimiter", "#{"],
57-
["number", "42"],
57+
["content", [
58+
["number", "42"]
59+
]],
5860
["delimiter", "}"]
5961
]],
60-
"\""
62+
["string", "\""]
6163
]],
6264
["punctuation", "}"]
6365
]]
@@ -68,8 +70,7 @@
6870
["attributes", [
6971
["punctuation", "{"],
7072
["symbol", ":id"],
71-
["operator", "="],
72-
["operator", ">"],
73+
["operator", "=>"],
7374
["punctuation", "["],
7475
["variable", "@item"],
7576
["punctuation", "."],
@@ -111,7 +112,9 @@
111112
["punctuation", "{"],
112113
"html_attrs",
113114
["punctuation", "("],
114-
["string", ["'fr-fr'"]],
115+
["string-literal", [
116+
["string", "'fr-fr'"]
117+
]],
115118
["punctuation", ")"],
116119
["punctuation", "}"]
117120
]]
@@ -142,8 +145,7 @@
142145
["attributes", [
143146
["punctuation", "{"],
144147
["symbol", ":id"],
145-
["operator", "="],
146-
["operator", ">"],
148+
["operator", "=>"],
147149
["variable", "@article"],
148150
["punctuation", "."],
149151
"number",

‎tests/languages/ruby+haml/ruby_inclusion.test

+6-10
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,12 @@
2020
["function", "circumference"]
2121
]],
2222

23-
["constant", "Math"],
24-
["punctuation", ":"],
25-
["punctuation", ":"],
23+
"\r\n\t\tMath",
24+
["double-colon", "::"],
2625
["constant", "PI"],
2726
["operator", "*"],
2827
" radius ",
29-
["operator", "*"],
30-
["operator", "*"],
28+
["operator", "**"],
3129
["number", "2"],
3230

3331
["keyword", "end"]
@@ -44,14 +42,12 @@
4442
["function", "circumference"]
4543
]],
4644

47-
["constant", "Math"],
48-
["punctuation", ":"],
49-
["punctuation", ":"],
45+
"\r\n\t\t\tMath",
46+
["double-colon", "::"],
5047
["constant", "PI"],
5148
["operator", "*"],
5249
" radius ",
53-
["operator", "*"],
54-
["operator", "*"],
50+
["operator", "**"],
5551
["number", "2"],
5652

5753
["keyword", "end"]

‎tests/languages/ruby/class-name_feature.test

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
class Customer
2+
@@no_of_customers = 0
3+
end
4+
5+
cust1 = Customer. new
6+
cust2 = Customer. new
7+
8+
class Accounts
9+
def reading_charge
10+
end
11+
def Accounts.return_date
12+
end
13+
end
14+
15+
class Salad
16+
def self.buy_olive_oil
17+
end
18+
end
19+
20+
----------------------------------------------------
21+
22+
[
23+
["keyword", "class"], ["class-name", ["Customer"]],
24+
["variable", "@@no_of_customers"], ["operator", "="], ["number", "0"],
25+
["keyword", "end"],
26+
27+
"\r\n\r\ncust1 ",
28+
["operator", "="],
29+
["class-name", ["Customer"]],
30+
["punctuation", "."],
31+
["keyword", "new"],
32+
33+
"\r\ncust2 ",
34+
["operator", "="],
35+
["class-name", ["Customer"]],
36+
["punctuation", "."],
37+
["keyword", "new"],
38+
39+
["keyword", "class"],
40+
["class-name", ["Accounts"]],
41+
42+
["keyword", "def"],
43+
["method-definition", [
44+
["function", "reading_charge"]
45+
]],
46+
47+
["keyword", "end"],
48+
49+
["keyword", "def"],
50+
["method-definition", [
51+
["class-name", "Accounts"],
52+
["punctuation", "."],
53+
["function", "return_date"]
54+
]],
55+
56+
["keyword", "end"],
57+
58+
["keyword", "end"],
59+
60+
["keyword", "class"],
61+
["class-name", ["Salad"]],
62+
63+
["keyword", "def"],
64+
["method-definition", [
65+
["keyword", "self"],
66+
["punctuation", "."],
67+
["function", "buy_olive_oil"]
68+
]],
69+
70+
["keyword", "end"],
71+
72+
["keyword", "end"]
73+
]

‎tests/languages/ruby/command_feature.test

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
`echo foo`
2+
`echo #{user_input}`
3+
`grep hosts /private/etc/* 2>&1`
4+
5+
%x[ ls ]
6+
%x{ ls }
7+
%x<ls -al #{dir}>
8+
9+
%x!foo #{ 42 }!
10+
%x(foo #{ 42 })
11+
%x{foo #{ 42 }}
12+
%x[foo #{ 42 }]
13+
%x<foo #{ 42 }>
14+
15+
----------------------------------------------------
16+
17+
[
18+
["command-literal", [
19+
["command", "`echo foo`"]
20+
]],
21+
["command-literal", [
22+
["command", "`echo "],
23+
["interpolation", [
24+
["delimiter", "#{"],
25+
["content", ["user_input"]],
26+
["delimiter", "}"]
27+
]],
28+
["command", "`"]
29+
]],
30+
["command-literal", [
31+
["command", "`grep hosts /private/etc/* 2>&1`"]
32+
]],
33+
34+
["command-literal", [
35+
["command", "%x[ ls ]"]
36+
]],
37+
["command-literal", [
38+
["command", "%x{ ls }"]
39+
]],
40+
["command-literal", [
41+
["command", "%x<ls -al "],
42+
["interpolation", [
43+
["delimiter", "#{"],
44+
["content", ["dir"]],
45+
["delimiter", "}"]
46+
]],
47+
["command", ">"]
48+
]],
49+
50+
["command-literal", [
51+
["command", "%x!foo "],
52+
["interpolation", [
53+
["delimiter", "#{"],
54+
["content", [
55+
["number", "42"]
56+
]],
57+
["delimiter", "}"]
58+
]],
59+
["command", "!"]
60+
]],
61+
["command-literal", [
62+
["command", "%x(foo "],
63+
["interpolation", [
64+
["delimiter", "#{"],
65+
["content", [
66+
["number", "42"]
67+
]],
68+
["delimiter", "}"]
69+
]],
70+
["command", ")"]
71+
]],
72+
["command-literal", [
73+
["command", "%x{foo "],
74+
["interpolation", [
75+
["delimiter", "#{"],
76+
["content", [
77+
["number", "42"]
78+
]],
79+
["delimiter", "}"]
80+
]],
81+
["command", "}"]
82+
]],
83+
["command-literal", [
84+
["command", "%x[foo "],
85+
["interpolation", [
86+
["delimiter", "#{"],
87+
["content", [
88+
["number", "42"]
89+
]],
90+
["delimiter", "}"]
91+
]],
92+
["command", "]"]
93+
]],
94+
["command-literal", [
95+
["command", "%x<foo "],
96+
["interpolation", [
97+
["delimiter", "#{"],
98+
["content", [
99+
["number", "42"]
100+
]],
101+
["delimiter", "}"]
102+
]],
103+
["command", ">"]
104+
]]
105+
]

‎tests/languages/ruby/constant_feature.test

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
Foobar
21
FOO_BAR_42
32
F
43
FOO
@@ -8,7 +7,6 @@ BAZ!
87
----------------------------------------------------
98

109
[
11-
["constant", "Foobar"],
1210
["constant", "FOO_BAR_42"],
1311
["constant", "F"],
1412
["constant", "FOO"],
@@ -18,4 +16,4 @@ BAZ!
1816

1917
----------------------------------------------------
2018

21-
Checks for constants.
19+
Checks for constants.

‎tests/languages/ruby/issue1336.test

+2-4
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@ Foo::Bar
55

66
[
77
["symbol", ":Foo"],
8-
["constant", "Foo"],
9-
["punctuation", ":"], ["punctuation", ":"],
10-
["constant", "Bar"]
8+
"\r\nFoo", ["double-colon", "::"], "Bar"
119
]
1210

1311
----------------------------------------------------
1412

15-
Ensures module syntax is not confused with symbols. See #1336
13+
Ensures module syntax is not confused with symbols. See #1336

‎tests/languages/ruby/keyword_feature.test

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ for
2020
if
2121
in
2222
include
23-
module
23+
module;
2424
new;
2525
next
2626
nil
@@ -72,7 +72,7 @@ yield
7272
["keyword", "if"],
7373
["keyword", "in"],
7474
["keyword", "include"],
75-
["keyword", "module"],
75+
["keyword", "module"], ["punctuation", ";"],
7676
["keyword", "new"], ["punctuation", ";"],
7777
["keyword", "next"],
7878
["keyword", "nil"],

‎tests/languages/ruby/method_definition_feature.test

+4-6
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,12 @@ end
5959
["function", "circumference"]
6060
]],
6161

62-
["constant", "Math"],
63-
["punctuation", ":"],
64-
["punctuation", ":"],
62+
"\r\n Math",
63+
["double-colon", "::"],
6564
["constant", "PI"],
6665
["operator", "*"],
6766
" radius ",
68-
["operator", "*"],
69-
["operator", "*"],
67+
["operator", "**"],
7068
["number", "2"],
7169

7270
["keyword", "end"],
@@ -78,7 +76,7 @@ end
7876
["function", "grow_by"]
7977
]],
8078
" factor",
81-
["punctuation", ":"],
79+
["operator", ":"],
8280

8381
["variable", "@radius"],
8482
["operator", "="],

‎tests/languages/ruby/operator_feature.test

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
+ - * / % **
2+
+= -= *= /= %= **=
3+
4+
== != < > <= >= <=> ===
5+
!~ =~
6+
=
7+
& | ^ ~ << >>
8+
&= |= ^= <<= >>=
9+
&& || !
10+
&&= ||=
11+
12+
=>
13+
14+
&.
15+
16+
? :
17+
.. ...
18+
19+
and or not
20+
21+
----------------------------------------------------
22+
23+
[
24+
["operator", "+"],
25+
["operator", "-"],
26+
["operator", "*"],
27+
["operator", "/"],
28+
["operator", "%"],
29+
["operator", "**"],
30+
31+
["operator", "+="],
32+
["operator", "-="],
33+
["operator", "*="],
34+
["operator", "/="],
35+
["operator", "%="],
36+
["operator", "**="],
37+
38+
["operator", "=="],
39+
["operator", "!="],
40+
["operator", "<"],
41+
["operator", ">"],
42+
["operator", "<="],
43+
["operator", ">="],
44+
["operator", "<=>"],
45+
["operator", "==="],
46+
47+
["operator", "!~"],
48+
["operator", "=~"],
49+
50+
["operator", "="],
51+
52+
["operator", "&"],
53+
["operator", "|"],
54+
["operator", "^"],
55+
["operator", "~"],
56+
["operator", "<<"],
57+
["operator", ">>"],
58+
59+
["operator", "&="],
60+
["operator", "|="],
61+
["operator", "^="],
62+
["operator", "<<="],
63+
["operator", ">>="],
64+
65+
["operator", "&&"],
66+
["operator", "||"],
67+
["operator", "!"],
68+
69+
["operator", "&&="],
70+
["operator", "||="],
71+
72+
["operator", "=>"],
73+
74+
["operator", "&."],
75+
76+
["operator", "?"], ["operator", ":"],
77+
["operator", ".."], ["operator", "..."],
78+
79+
["keyword", "and"], ["keyword", "or"], ["keyword", "not"]
80+
]

‎tests/languages/ruby/punctuation_feature.test

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
( ) { } [ ]
2+
. , ;
3+
::
4+
5+
----------------------------------------------------
6+
7+
[
8+
["punctuation", "("],
9+
["punctuation", ")"],
10+
["punctuation", "{"],
11+
["punctuation", "}"],
12+
["punctuation", "["],
13+
["punctuation", "]"],
14+
15+
["punctuation", "."],
16+
["punctuation", ","],
17+
["punctuation", ";"],
18+
19+
["double-colon", "::"]
20+
]

‎tests/languages/ruby/regex_feature.test

+90-52
Original file line numberDiff line numberDiff line change
@@ -26,111 +26,149 @@
2626
----------------------------------------------------
2727

2828
[
29-
["regex", ["/[foo]\\/bar/gim"]],
29+
["regex-literal", [
30+
["regex", "/[foo]\\/bar/gim"]
31+
]],
3032

31-
["regex", ["/[bar]/"]],
33+
["regex-literal", [
34+
["regex", "/[bar]/"]
35+
]],
3236
["punctuation", ","],
3337

34-
["regex", ["/./i"]],
38+
["regex-literal", [
39+
["regex", "/./i"]
40+
]],
3541
["punctuation", ";"],
3642

37-
["regex", [
38-
"/foo",
43+
["regex-literal", [
44+
["regex", "/foo"],
3945
["interpolation", [
4046
["delimiter", "#{"],
41-
"bar",
47+
["content", ["bar"]],
4248
["delimiter", "}"]
4349
]],
44-
"/"
50+
["regex", "/"]
4551
]],
4652
["punctuation", ";"],
4753

48-
["regex", ["/ab+c/ix"]],
54+
["regex-literal", [
55+
["regex", "/ab+c/ix"]
56+
]],
4957

50-
["regex", [
51-
"%r!foo?bar",
58+
["regex-literal", [
59+
["regex", "%r!foo?bar"],
5260
["interpolation", [
5361
["delimiter", "#{"],
54-
["number", "39"],
55-
["operator", "+"],
56-
["number", "3"],
62+
["content", [
63+
["number", "39"],
64+
["operator", "+"],
65+
["number", "3"]
66+
]],
5767
["delimiter", "}"]
5868
]],
59-
"!"
69+
["regex", "!"]
6070
]],
6171

62-
["regex", [
63-
"%r(foo?bar",
72+
["regex-literal", [
73+
["regex", "%r(foo?bar"],
6474
["interpolation", [
6575
["delimiter", "#{"],
66-
["number", "39"],
67-
["operator", "+"],
68-
["number", "3"],
76+
["content", [
77+
["number", "39"],
78+
["operator", "+"],
79+
["number", "3"]
80+
]],
6981
["delimiter", "}"]
7082
]],
71-
")"
83+
["regex", ")"]
7284
]],
7385

74-
["regex", [
75-
"%r{foo?bar",
86+
["regex-literal", [
87+
["regex", "%r{foo?bar"],
7688
["interpolation", [
7789
["delimiter", "#{"],
78-
["number", "39"],
79-
["operator", "+"],
80-
["number", "3"],
90+
["content", [
91+
["number", "39"],
92+
["operator", "+"],
93+
["number", "3"]
94+
]],
8195
["delimiter", "}"]
8296
]],
83-
"}"
97+
["regex", "}"]
8498
]],
8599

86-
["regex", [
87-
"%r[foo?bar",
100+
["regex-literal", [
101+
["regex", "%r[foo?bar"],
88102
["interpolation", [
89103
["delimiter", "#{"],
90-
["number", "39"],
91-
["operator", "+"],
92-
["number", "3"],
104+
["content", [
105+
["number", "39"],
106+
["operator", "+"],
107+
["number", "3"]
108+
]],
93109
["delimiter", "}"]
94110
]],
95-
"]"
111+
["regex", "]"]
96112
]],
97113

98-
["regex", [
99-
"%r<foo?bar",
114+
["regex-literal", [
115+
["regex", "%r<foo?bar"],
100116
["interpolation", [
101117
["delimiter", "#{"],
102-
["number", "39"],
103-
["operator", "+"],
104-
["number", "3"],
118+
["content", [
119+
["number", "39"],
120+
["operator", "+"],
121+
["number", "3"]
122+
]],
105123
["delimiter", "}"]
106124
]],
107-
">"
125+
["regex", ">"]
108126
]],
109127

110-
["regex", ["/foo/"]],
128+
["regex-literal", [
129+
["regex", "/foo/"]
130+
]],
111131
["comment", "# comment"],
112132

113-
["regex", [
114-
"/foo",
133+
["regex-literal", [
134+
["regex", "/foo"],
115135
["interpolation", [
116136
["delimiter", "#{"],
117-
"bar",
137+
["content", ["bar"]],
118138
["delimiter", "}"]
119139
]],
120-
"/"
140+
["regex", "/"]
121141
]],
122142
["comment", "# comment"],
123143

124144
["comment", "# flags"],
125-
["regex", ["/abc/e"]],
126-
["regex", ["/abc/g"]],
127-
["regex", ["/abc/i"]],
128-
["regex", ["/abc/m"]],
129-
["regex", ["/abc/n"]],
130-
["regex", ["/abc/o"]],
131-
["regex", ["/abc/s"]],
132-
["regex", ["/abc/u"]],
133-
["regex", ["/abc/x"]]
145+
["regex-literal", [
146+
["regex", "/abc/e"]
147+
]],
148+
["regex-literal", [
149+
["regex", "/abc/g"]
150+
]],
151+
["regex-literal", [
152+
["regex", "/abc/i"]
153+
]],
154+
["regex-literal", [
155+
["regex", "/abc/m"]
156+
]],
157+
["regex-literal", [
158+
["regex", "/abc/n"]
159+
]],
160+
["regex-literal", [
161+
["regex", "/abc/o"]
162+
]],
163+
["regex-literal", [
164+
["regex", "/abc/s"]
165+
]],
166+
["regex-literal", [
167+
["regex", "/abc/u"]
168+
]],
169+
["regex-literal", [
170+
["regex", "/abc/x"]
171+
]]
134172
]
135173

136174
----------------------------------------------------

‎tests/languages/ruby/string_feature.test

+346-315
Large diffs are not rendered by default.

‎tests/languages/ruby/symbol_feature.test

+65-2
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,79 @@
22
:foo
33
:BAR?
44
:Baz_42!
5+
:あ
6+
:"name"
7+
:"\u{c4 d6 dc}"
8+
:question?
9+
:exclamation!
10+
:$;
11+
12+
:foo.object_id
13+
14+
# in hashes
15+
16+
{ :one => "eins", :two => "zwei", :three => "drei" }
17+
{ one: "eins", two: "zwei", three: "drei" }
518

619
----------------------------------------------------
720

821
[
922
["symbol", ":_"],
1023
["symbol", ":foo"],
1124
["symbol", ":BAR?"],
12-
["symbol", ":Baz_42!"]
25+
["symbol", ":Baz_42!"],
26+
["symbol", ":あ"],
27+
["symbol", ":\"name\""],
28+
["symbol", ":\"\\u{c4 d6 dc}\""],
29+
["symbol", ":question?"],
30+
["symbol", ":exclamation!"],
31+
["symbol", ":$;"],
32+
33+
["symbol", ":foo"], ["punctuation", "."], "object_id\r\n\r\n",
34+
35+
["comment", "# in hashes"],
36+
37+
["punctuation", "{"],
38+
["symbol", ":one"],
39+
["operator", "=>"],
40+
["string-literal", [
41+
["string", "\"eins\""]
42+
]],
43+
["punctuation", ","],
44+
["symbol", ":two"],
45+
["operator", "=>"],
46+
["string-literal", [
47+
["string", "\"zwei\""]
48+
]],
49+
["punctuation", ","],
50+
["symbol", ":three"],
51+
["operator", "=>"],
52+
["string-literal", [
53+
["string", "\"drei\""]
54+
]],
55+
["punctuation", "}"],
56+
57+
["punctuation", "{"],
58+
["symbol", "one"],
59+
["operator", ":"],
60+
["string-literal", [
61+
["string", "\"eins\""]
62+
]],
63+
["punctuation", ","],
64+
["symbol", "two"],
65+
["operator", ":"],
66+
["string-literal", [
67+
["string", "\"zwei\""]
68+
]],
69+
["punctuation", ","],
70+
["symbol", "three"],
71+
["operator", ":"],
72+
["string-literal", [
73+
["string", "\"drei\""]
74+
]],
75+
["punctuation", "}"]
1376
]
1477

1578
----------------------------------------------------
1679

17-
Checks for symbols.
80+
Checks for symbols.

0 commit comments

Comments
 (0)
Please sign in to comment.