Skip to content

Commit d04d166

Browse files
Structured text: Improved tokenization (#3213)
1 parent dfbb202 commit d04d166

6 files changed

+87
-16
lines changed

components/prism-iecst.js

+13-11
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ Prism.languages.iecst = {
33
{
44
pattern: /(^|[^\\])(?:\/\*[\s\S]*?(?:\*\/|$)|\(\*[\s\S]*?(?:\*\)|$)|\{[\s\S]*?(?:\}|$))/,
55
lookbehind: true,
6+
greedy: true,
67
},
78
{
89
pattern: /(^|[^\\:])\/\/.*/,
@@ -14,17 +15,18 @@ Prism.languages.iecst = {
1415
pattern: /(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
1516
greedy: true,
1617
},
17-
'class-name': /\b(?:END_)?(?:PROGRAM|CONFIGURATION|INTERFACE|FUNCTION_BLOCK|FUNCTION|ACTION|TRANSITION|TYPE|STRUCT|(?:INITIAL_)?STEP|NAMESPACE|LIBRARY|CHANNEL|FOLDER|RESOURCE|VAR_(?:ACCESS|CONFIG|EXTERNAL|GLOBAL|INPUT|IN_OUT|PUTPUT|TEMP)|VAR|METHOD|PROPERTY)\b/i,
18-
'keyword': /\b(?:AT|BY|(?:END_)?(?:CASE|FOR|IF|REPEAT|WHILE)|CONSTANT|CONTINUE|DO|ELSE|ELSIF|EXIT|EXTENDS|FROM|GET|GOTO|IMPLEMENTS|JMP|NON_RETAIN|PRIVATE|PROTECTED|PUBLIC|RETAIN|RETURN|SET|TASK|THEN|TO|UNTIL|USING|WITH|__CATCH|__ENDTRY|__FINALLY|__TRY)\b/,
19-
'variable': /\b(?:ANY|ARRAY|AT|BOOL|BYTE|U?(?:D|L|S)?INT|(?:D|L)?WORD|DATE(?:_AND_TIME)?|DT|L?REAL|POINTER|STRING|TIME(?:_OF_DAY)?|TOD)\b/,
20-
'symbol': /%[IQM][XBWDL][\d.]*|%[IQ][\d.]*/,
18+
'keyword': [
19+
/\b(?:END_)?(?:PROGRAM|CONFIGURATION|INTERFACE|FUNCTION_BLOCK|FUNCTION|ACTION|TRANSITION|TYPE|STRUCT|(?:INITIAL_)?STEP|NAMESPACE|LIBRARY|CHANNEL|FOLDER|RESOURCE|VAR_(?:ACCESS|CONFIG|EXTERNAL|GLOBAL|INPUT|IN_OUT|OUTPUT|TEMP)|VAR|METHOD|PROPERTY)\b/i,
20+
/\b(?:AT|BY|(?:END_)?(?:CASE|FOR|IF|REPEAT|WHILE)|CONSTANT|CONTINUE|DO|ELSE|ELSIF|EXIT|EXTENDS|FROM|GET|GOTO|IMPLEMENTS|JMP|NON_RETAIN|OF|PRIVATE|PROTECTED|PUBLIC|RETAIN|RETURN|SET|TASK|THEN|TO|UNTIL|USING|WITH|__CATCH|__ENDTRY|__FINALLY|__TRY)\b/
21+
],
22+
'class-name': /\b(?:ANY|ARRAY|BOOL|BYTE|U?(?:D|L|S)?INT|(?:D|L)?WORD|DATE(?:_AND_TIME)?|DT|L?REAL|POINTER|STRING|TIME(?:_OF_DAY)?|TOD)\b/,
23+
'address': {
24+
pattern: /%[IQM][XBWDL][\d.]*|%[IQ][\d.]*/,
25+
alias: 'symbol'
26+
},
2127
'number': /\b(?:16#[\da-f]+|2#[01_]+|0x[\da-f]+)\b|\b(?:D|DT|T|TOD)#[\d_shmd:]*|\b[A-Z]*#[\d.,_]*|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,
2228
'boolean': /\b(?:FALSE|NULL|TRUE)\b/,
23-
'function': /\w+(?=\()/,
24-
'operator': /(?:S?R?:?=>?|&&?|\*\*?|<=?|>=?|[-:^/+])|\b(?:AND|EQ|GE|GT|LE|LT|MOD|NE|NOT|OR|XOR)\b/,
25-
'punctuation': /[();]/,
26-
'type': {
27-
'pattern': /#/,
28-
'alias': 'selector',
29-
},
29+
'operator': /S?R?:?=>?|&&?|\*\*?|<[=>]?|>=?|[-:^/+#]|\b(?:AND|EQ|EXPT|GE|GT|LE|LT|MOD|NE|NOT|OR|XOR)\b/,
30+
'function': /\b[a-z_]\w*(?=\s*\()/i,
31+
'punctuation': /[()[\].,;]/,
3032
};

components/prism-iecst.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

tests/languages/iecst/symbol.test tests/languages/iecst/address_feature.test

+4-4
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,18 @@ END_VAR
55
----------------------------------------------------
66

77
[
8-
["class-name", "VAR"],
8+
["keyword", "VAR"],
99

1010
"\r\n varname ",
1111
["keyword", "AT"],
12-
["symbol", "%QX1.0.0"],
12+
["address", "%QX1.0.0"],
1313
["operator", ":"],
14-
["variable", "BOOL"],
14+
["class-name", "BOOL"],
1515
["operator", ":="],
1616
["boolean", "TRUE"],
1717
["punctuation", ";"],
1818

19-
["class-name", "END_VAR"]
19+
["keyword", "END_VAR"]
2020
]
2121

2222
----------------------------------------------------
File renamed without changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
= <> < <= > >=
2+
+ - * / ** ^ & &&
3+
4+
: :=
5+
#
6+
7+
AND
8+
EQ
9+
EXPT
10+
GE
11+
GT
12+
LE
13+
LT
14+
MOD
15+
NE
16+
NOT
17+
OR
18+
XOR
19+
20+
----------------------------------------------------
21+
22+
[
23+
["operator", "="],
24+
["operator", "<>"],
25+
["operator", "<"],
26+
["operator", "<="],
27+
["operator", ">"],
28+
["operator", ">="],
29+
30+
["operator", "+"],
31+
["operator", "-"],
32+
["operator", "*"],
33+
["operator", "/"],
34+
["operator", "**"],
35+
["operator", "^"],
36+
["operator", "&"],
37+
["operator", "&&"],
38+
39+
["operator", ":"], ["operator", ":="],
40+
["operator", "#"],
41+
42+
["operator", "AND"],
43+
["operator", "EQ"],
44+
["operator", "EXPT"],
45+
["operator", "GE"],
46+
["operator", "GT"],
47+
["operator", "LE"],
48+
["operator", "LT"],
49+
["operator", "MOD"],
50+
["operator", "NE"],
51+
["operator", "NOT"],
52+
["operator", "OR"],
53+
["operator", "XOR"]
54+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
( ) [ ]
2+
, ; .
3+
4+
----------------------------------------------------
5+
6+
[
7+
["punctuation", "("],
8+
["punctuation", ")"],
9+
["punctuation", "["],
10+
["punctuation", "]"],
11+
12+
["punctuation", ","],
13+
["punctuation", ";"],
14+
["punctuation", "."]
15+
]

0 commit comments

Comments
 (0)