Skip to content

Commit baa95ca

Browse files
RunDevelopmentRunDevelopment
authoredDec 5, 2021
Keyman: Improved tokenization (#3224)
·
v1.30.0v1.26.0
1 parent d04d166 commit baa95ca

15 files changed

+291
-173
lines changed
 

‎components/prism-keyman.js

+42-12
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,44 @@
11
Prism.languages.keyman = {
2-
'comment': /\bc\s.*/i,
3-
'function': /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|".+?"|'.+?')\s*\]/i, // virtual key
4-
'string': /("|').*?\1/,
5-
'bold': [ // header statements, system stores and variable system stores
6-
/&(?:baselayout|bitmap|capsalwaysoff|capsononly|copyright|ethnologuecode|hotkey|includecodes|keyboardversion|kmw_embedcss|kmw_embedjs|kmw_helpfile|kmw_helptext|kmw_rtl|language|layer|layoutfile|message|mnemoniclayout|name|oldcharposmatching|platform|shiftfreescaps|targets|version|visualkeyboard|windowslanguages)\b/i,
7-
/\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i
8-
],
9-
'keyword': /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i, // rule keywords
10-
'atrule': /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i, // structural keywords
11-
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i, // U+####, x###, d### characters and numbers
12-
'operator': /[+>\\,()]/,
13-
'tag': /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i // prefixes
2+
'comment': {
3+
pattern: /\bc .*/i,
4+
greedy: true
5+
},
6+
'string': {
7+
pattern: /"[^"\r\n]*"|'[^'\r\n]*'/,
8+
greedy: true
9+
},
10+
'virtual-key': {
11+
pattern: /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|[A-E]\d\d?|"[^"\r\n]*"|'[^'\r\n]*')\s*\]/i,
12+
greedy: true,
13+
alias: 'function' // alias for styles
14+
},
15+
16+
// https://help.keyman.com/developer/language/guide/headers
17+
'header-keyword': {
18+
pattern: /&\w+/,
19+
alias: 'bold' // alias for styles
20+
},
21+
'header-statement': {
22+
pattern: /\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i,
23+
alias: 'bold' // alias for styles
24+
},
25+
26+
'rule-keyword': {
27+
pattern: /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i,
28+
alias: 'keyword'
29+
},
30+
'structural-keyword': {
31+
pattern: /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i,
32+
alias: 'keyword'
33+
},
34+
35+
'compile-target': {
36+
pattern: /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i,
37+
alias: 'property'
38+
},
39+
40+
// U+####, x###, d### characters and numbers
41+
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i,
42+
'operator': /[+>\\$]|\.\./,
43+
'punctuation': /[()=,]/
1444
};

‎components/prism-keyman.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/keyman/atrule_feature.test

-15
This file was deleted.

‎tests/languages/keyman/bold_feature.test

-59
This file was deleted.

‎tests/languages/keyman/compile-target_feature.test

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
$keyman:
2+
$kmfl:
3+
$weaver:
4+
$keymanweb:
5+
$keymanonly:
6+
7+
c Note: return statement is only supported in .kmx in Keyman 13
8+
$keymanonly: + 'a' > 'a' return
9+
10+
----------------------------------------------------
11+
12+
[
13+
["compile-target", "$keyman:"],
14+
["compile-target", "$kmfl:"],
15+
["compile-target", "$weaver:"],
16+
["compile-target", "$keymanweb:"],
17+
["compile-target", "$keymanonly:"],
18+
19+
["comment", "c Note: return statement is only supported in .kmx in Keyman 13"],
20+
21+
["compile-target", "$keymanonly:"],
22+
["operator", "+"],
23+
["string", "'a'"],
24+
["operator", ">"],
25+
["string", "'a'"],
26+
["rule-keyword", "return"]
27+
]

‎tests/languages/keyman/function_feature.test

-39
This file was deleted.

‎tests/languages/keyman/header-keyword.test

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
&baselayout
2+
&bitmap
3+
&capsononly
4+
&capsalwaysoff
5+
&shiftfreescaps
6+
&copyright
7+
&ethnologuecode
8+
&hotkey
9+
&includecodes
10+
&keyboardversion
11+
&kmw_embedcss
12+
&kmw_embedjs
13+
&kmw_helpfile
14+
&kmw_helptext
15+
&kmw_rtl
16+
&language
17+
&layer
18+
&layoutfile
19+
&message
20+
&mnemoniclayout
21+
&name
22+
&oldcharposmatching
23+
&platform
24+
&targets
25+
&version
26+
&visualkeyboard
27+
&windowslanguages
28+
29+
----------------------------------------------------
30+
31+
[
32+
["header-keyword", "&baselayout"],
33+
["header-keyword", "&bitmap"],
34+
["header-keyword", "&capsononly"],
35+
["header-keyword", "&capsalwaysoff"],
36+
["header-keyword", "&shiftfreescaps"],
37+
["header-keyword", "&copyright"],
38+
["header-keyword", "&ethnologuecode"],
39+
["header-keyword", "&hotkey"],
40+
["header-keyword", "&includecodes"],
41+
["header-keyword", "&keyboardversion"],
42+
["header-keyword", "&kmw_embedcss"],
43+
["header-keyword", "&kmw_embedjs"],
44+
["header-keyword", "&kmw_helpfile"],
45+
["header-keyword", "&kmw_helptext"],
46+
["header-keyword", "&kmw_rtl"],
47+
["header-keyword", "&language"],
48+
["header-keyword", "&layer"],
49+
["header-keyword", "&layoutfile"],
50+
["header-keyword", "&message"],
51+
["header-keyword", "&mnemoniclayout"],
52+
["header-keyword", "&name"],
53+
["header-keyword", "&oldcharposmatching"],
54+
["header-keyword", "&platform"],
55+
["header-keyword", "&targets"],
56+
["header-keyword", "&version"],
57+
["header-keyword", "&visualkeyboard"],
58+
["header-keyword", "&windowslanguages"]
59+
]

‎tests/languages/keyman/header-statement.test

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
bitmap
2+
bitmaps
3+
caps on only
4+
caps always off
5+
shift frees caps
6+
copyright
7+
hotkey
8+
language
9+
layout
10+
message
11+
name
12+
version
13+
14+
----------------------------------------------------
15+
16+
[
17+
["header-statement", "bitmap"],
18+
["header-statement", "bitmaps"],
19+
["header-statement", "caps on only"],
20+
["header-statement", "caps always off"],
21+
["header-statement", "shift frees caps"],
22+
["header-statement", "copyright"],
23+
["header-statement", "hotkey"],
24+
["header-statement", "language"],
25+
["header-statement", "layout"],
26+
["header-statement", "message"],
27+
["header-statement", "name"],
28+
["header-statement", "version"]
29+
]

‎tests/languages/keyman/keyword_feature.test

-21
This file was deleted.

‎tests/languages/keyman/operator_feature.test

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
+ >
2-
\ ,
3-
( )
1+
+ > \
2+
3+
$vowel_a
4+
'a' .. 'z'
45

56
----------------------------------------------------
67

78
[
8-
["operator", "+"], ["operator", ">"],
9-
["operator", "\\"], ["operator", ","],
10-
["operator", "("], ["operator", ")"]
9+
["operator", "+"], ["operator", ">"], ["operator", "\\"],
10+
11+
["operator", "$"], "vowel_a\r\n",
12+
["string", "'a'"], ["operator", ".."], ["string", "'z'"]
1113
]
1214

1315
----------------------------------------------------
1416

15-
Checks for all operators.
17+
Checks for all operators.

‎tests/languages/keyman/punctuation_feature.test

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
( )
2+
, =
3+
4+
----------------------------------------------------
5+
6+
[
7+
["punctuation", "("], ["punctuation", ")"],
8+
["punctuation", ","], ["punctuation", "="]
9+
]

‎tests/languages/keyman/rule-keyword_feature.test

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
any
2+
baselayout
3+
beep
4+
call
5+
context
6+
deadkey
7+
dk
8+
if
9+
index
10+
layer
11+
notany
12+
nul
13+
outs
14+
platform
15+
reset
16+
return
17+
save
18+
set
19+
store
20+
use
21+
22+
----------------------------------------------------
23+
24+
[
25+
["rule-keyword", "any"],
26+
["rule-keyword", "baselayout"],
27+
["rule-keyword", "beep"],
28+
["rule-keyword", "call"],
29+
["rule-keyword", "context"],
30+
["rule-keyword", "deadkey"],
31+
["rule-keyword", "dk"],
32+
["rule-keyword", "if"],
33+
["rule-keyword", "index"],
34+
["rule-keyword", "layer"],
35+
["rule-keyword", "notany"],
36+
["rule-keyword", "nul"],
37+
["rule-keyword", "outs"],
38+
["rule-keyword", "platform"],
39+
["rule-keyword", "reset"],
40+
["rule-keyword", "return"],
41+
["rule-keyword", "save"],
42+
["rule-keyword", "set"],
43+
["rule-keyword", "store"],
44+
["rule-keyword", "use"]
45+
]
46+
47+
----------------------------------------------------
48+
49+
Checks for all keywords.

‎tests/languages/keyman/structural-keyword_feature.test

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
ansi
2+
begin
3+
unicode
4+
group
5+
using keys
6+
match
7+
nomatch
8+
9+
----------------------------------------------------
10+
11+
[
12+
["structural-keyword", "ansi"],
13+
["structural-keyword", "begin"],
14+
["structural-keyword", "unicode"],
15+
["structural-keyword", "group"],
16+
["structural-keyword", "using keys"],
17+
["structural-keyword", "match"],
18+
["structural-keyword", "nomatch"]
19+
]
20+
21+
----------------------------------------------------
22+
23+
Checks for all structural keywords.

‎tests/languages/keyman/tag_feature.test

-19
This file was deleted.

‎tests/languages/keyman/virtual-key_feature.test

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
['c']
2+
["f"]
3+
[K_SEL]
4+
[K_KANJI?15]
5+
[T_SCHWA]
6+
[U_0259]
7+
[CTRL 'a']
8+
[SHIFT "b"]
9+
[ALT K_C]
10+
[LCTRL T_D]
11+
[RCTRL U_E259]
12+
[LALT K_F]
13+
[RALT K_G]
14+
[CAPS K_H]
15+
[NCAPS K_I]
16+
[SHIFT CAPS K_A]
17+
[SHIFT C01]
18+
19+
----------------------------------------------------
20+
21+
[
22+
["virtual-key", "['c']"],
23+
["virtual-key", "[\"f\"]"],
24+
["virtual-key", "[K_SEL]"],
25+
["virtual-key", "[K_KANJI?15]"],
26+
["virtual-key", "[T_SCHWA]"],
27+
["virtual-key", "[U_0259]"],
28+
["virtual-key", "[CTRL 'a']"],
29+
["virtual-key", "[SHIFT \"b\"]"],
30+
["virtual-key", "[ALT K_C]"],
31+
["virtual-key", "[LCTRL T_D]"],
32+
["virtual-key", "[RCTRL U_E259]"],
33+
["virtual-key", "[LALT K_F]"],
34+
["virtual-key", "[RALT K_G]"],
35+
["virtual-key", "[CAPS K_H]"],
36+
["virtual-key", "[NCAPS K_I]"],
37+
["virtual-key", "[SHIFT CAPS K_A]"],
38+
["virtual-key", "[SHIFT C01]"]
39+
]
40+
41+
----------------------------------------------------
42+
43+
Checks for keys, with all possible modifiers.

0 commit comments

Comments
 (0)
Please sign in to comment.