Jolie: Improved tokenization (#3221)

Author: RunDevelopment

components/prism-jolie.js

@@ -1,55 +1,41 @@
 Prism.languages.jolie = Prism.languages.extend('clike', {
 	'string': {
-		pattern: /(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
+		pattern: /(^|[^\\])"(?:\\[\s\S]|[^"\\])*"/,
+		lookbehind: true,
 		greedy: true
 	},
-	'keyword': /\b(?:Aggregates|Interfaces|Java|Javascript|Jolie|Location|OneWay|Protocol|Redirects|RequestResponse|cH|comp|concurrent|constants|courier|cset|csets|default|define|else|embedded|execution|exit|extender|for|foreach|forward|global|if|in|include|init|inputPort|install|instanceof|interface|is_defined|linkIn|linkOut|main|new|nullProcess|outputPort|over|provide|scope|sequential|service|single|spawn|synchronized|this|throw|throws|type|undef|until|while|with)\b/,
+	'class-name': {
+		pattern: /((?:\b(?:as|courier|embed|in|inputPort|outputPort|service)\b|@)[ \t]*)\w+/,
+		lookbehind: true
+	},
+	'keyword': /\b(?:as|cH|comp|concurrent|constants|courier|cset|csets|default|define|else|embed|embedded|execution|exit|extender|for|foreach|forward|from|global|if|import|in|include|init|inputPort|install|instanceof|interface|is_defined|linkIn|linkOut|main|new|nullProcess|outputPort|over|private|provide|public|scope|sequential|service|single|spawn|synchronized|this|throw|throws|type|undef|until|while|with)\b/,
+	'function': /\b[a-z_]\w*(?=[ \t]*[@(])/i,
 	'number': /(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?l?/i,
-	'operator': /-[-=>]?|\+[+=]?|<[<=]?|[>=*!]=?|&&|\|\||[:?\/%^]/,
-	'punctuation': /[,.]/,
-	'builtin': /\b(?:Byte|any|bool|char|double|float|int|long|string|undefined|void)\b/,
-	'symbol': /[|;@]/
+	'operator': /-[-=>]?|\+[+=]?|<[<=]?|[>=*!]=?|&&|\|\||[?\/%^@|]/,
+	'punctuation': /[()[\]{},;.:]/,
+	'builtin': /\b(?:Byte|any|bool|char|double|enum|float|int|length|long|ranges|regex|string|undefined|void)\b/
 });
 
-delete Prism.languages.jolie['class-name'];
-
 Prism.languages.insertBefore('jolie', 'keyword', {
-	'function':
-	{
-		pattern: /((?:\b(?:courier|in|inputPort|outputPort|service)\b|@)\s*)\w+/,
-		lookbehind: true
-	},
 	'aggregates': {
 		pattern: /(\bAggregates\s*:\s*)(?:\w+(?:\s+with\s+\w+)?\s*,\s*)*\w+(?:\s+with\s+\w+)?/,
 		lookbehind: true,
 		inside: {
-			'with-extension': {
-				pattern: /\bwith\s+\w+/,
-				inside: {
-					'keyword': /\bwith\b/
-				}
-			},
-			'function': {
-				pattern: /\w+/
-			},
-			'punctuation': {
-				pattern: /,/
-			}
+			'keyword': /\bwith\b/,
+			'class-name': /\w+/,
+			'punctuation': /,/
 		}
 	},
 	'redirects': {
 		pattern: /(\bRedirects\s*:\s*)(?:\w+\s*=>\s*\w+\s*,\s*)*(?:\w+\s*=>\s*\w+)/,
 		lookbehind: true,
 		inside: {
-			'punctuation': {
-				pattern: /,/
-			},
-			'function': {
-				pattern: /\w+/
-			},
-			'symbol': {
-				pattern: /=>/
-			}
+			'punctuation': /,/,
+			'class-name': /\w+/,
+			'operator': /=>/
 		}
+	},
+	'property': {
+		pattern: /\b(?:Aggregates|[Ii]nterfaces|Java|Javascript|Jolie|[Ll]ocation|OneWay|[Pp]rotocol|Redirects|RequestResponse)\b(?=[ \t]*:)/
 	}
 });

components/prism-jolie.min.js

@@ -1,27 +1,35 @@
-undefined
-string
-int
-void
-long
 Byte
+any
 bool
+char
 double
+enum
 float
-char
-any
+int
+length
+long
+ranges
+regex
+string
+undefined
+void
 
 ----------------------------------------------------
 
 [
-	["builtin", "undefined"],
-	["builtin", "string"],
-	["builtin", "int"],
-	["builtin", "void"],
-	["builtin", "long"],
 	["builtin", "Byte"],
+	["builtin", "any"],
 	["builtin", "bool"],
+	["builtin", "char"],
 	["builtin", "double"],
+	["builtin", "enum"],
 	["builtin", "float"],
-	["builtin", "char"],
-	["builtin", "any"]
+	["builtin", "int"],
+	["builtin", "length"],
+	["builtin", "long"],
+	["builtin", "ranges"],
+	["builtin", "regex"],
+	["builtin", "string"],
+	["builtin", "undefined"],
+	["builtin", "void"]
 ]

tests/languages/jolie/builtin_feature.test

@@ -0,0 +1,14 @@
+// single line
+
+/*
+multiple
+lines
+*/
+
+----------------------------------------------------
+
+[
+	["comment", "// single line"],
+
+	["comment", "/*\r\nmultiple\r\nlines\r\n*/"]
+]

tests/languages/jolie/comment_feature.test

@@ -1,56 +1,152 @@
 Aggregates: First, Second with Third
+
 Redirects: First => Second, Third => Fourth
+
 Jolie: "logger.ol" in LoggerService
+
 log@LoggerService( new )();
 println @ Console( "none" )()
 
+outputPort OutputPort3 {
+    location: "socket://localhost:9002/"
+    protocol: sodep
+    interfaces: Interface3
+}
+
+interface MyInterface {
+OneWay:
+    myOW( string )
+RequestResponse:
+    myRR( string )( string )
+}
+
+private service MainService {
+    embed ConfigurationService( ) as Conf
+    main {
+        getDBConn@Conf( )( res )
+    }
+}
+
 ----------------------------------------------------
 
 [
-	["keyword", "Aggregates"],
-	["operator", ":"],
+	["property", "Aggregates"],
+	["punctuation", ":"],
 	["aggregates", [
-		["function", "First"],
+		["class-name", "First"],
 		["punctuation", ","],
-		["function", "Second"],
-		["with-extension", [
-			["keyword", "with"],
-			" Third"
-		]]
+		["class-name", "Second"],
+		["keyword", "with"],
+		["class-name", "Third"]
 	]],
 
-	["keyword", "Redirects"],
-	["operator", ":"],
+	["property", "Redirects"],
+	["punctuation", ":"],
 	["redirects", [
-		["function", "First"],
-		["symbol", "=>"],
-		["function", "Second"],
+		["class-name", "First"],
+		["operator", "=>"],
+		["class-name", "Second"],
 		["punctuation", ","],
-		["function", "Third"],
-		["symbol", "=>"],
-		["function", "Fourth"]
+		["class-name", "Third"],
+		["operator", "=>"],
+		["class-name", "Fourth"]
 	]],
 
-	["keyword", "Jolie"],
-	["operator", ":"],
+	["property", "Jolie"],
+	["punctuation", ":"],
 	["string", "\"logger.ol\""],
 	["keyword", "in"],
-	["function", "LoggerService"],
+	["class-name", "LoggerService"],
 
-	"\r\nlog",
-	["symbol", "@"],
-	["function", "LoggerService"],
-	"( ",
+	["function", "log"],
+	["operator", "@"],
+	["class-name", "LoggerService"],
+	["punctuation", "("],
 	["keyword", "new"],
-	" )()",
-	["symbol", ";"],
+	["punctuation", ")"],
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", ";"],
 
-	"\r\nprintln ",
-	["symbol", "@"],
-	["function", "Console"],
-	"( ",
+	["function", "println"],
+	["operator", "@"],
+	["class-name", "Console"],
+	["punctuation", "("],
 	["string", "\"none\""],
-	" )()"
+	["punctuation", ")"],
+	["punctuation", "("],
+	["punctuation", ")"],
+
+	["keyword", "outputPort"],
+	["class-name", "OutputPort3"],
+	["punctuation", "{"],
+
+	["property", "location"],
+	["punctuation", ":"],
+	["string", "\"socket://localhost:9002/\""],
+
+	["property", "protocol"],
+	["punctuation", ":"],
+	" sodep\r\n    ",
+
+	["property", "interfaces"],
+	["punctuation", ":"],
+	" Interface3\r\n",
+
+	["punctuation", "}"],
+
+	["keyword", "interface"],
+	" MyInterface ",
+	["punctuation", "{"],
+
+	["property", "OneWay"],
+	["punctuation", ":"],
+
+	["function", "myOW"],
+	["punctuation", "("],
+	["builtin", "string"],
+	["punctuation", ")"],
+
+	["property", "RequestResponse"],
+	["punctuation", ":"],
+
+	["function", "myRR"],
+	["punctuation", "("],
+	["builtin", "string"],
+	["punctuation", ")"],
+	["punctuation", "("],
+	["builtin", "string"],
+	["punctuation", ")"],
+
+	["punctuation", "}"],
+
+	["keyword", "private"],
+	["keyword", "service"],
+	["class-name", "MainService"],
+	["punctuation", "{"],
+
+	["keyword", "embed"],
+	["class-name", "ConfigurationService"],
+	["punctuation", "("],
+	["punctuation", ")"],
+	["keyword", "as"],
+	["class-name", "Conf"],
+
+	["keyword", "main"],
+	["punctuation", "{"],
+
+	["function", "getDBConn"],
+	["operator", "@"],
+	["class-name", "Conf"],
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", "("],
+	" res ",
+	["punctuation", ")"],
+
+	["punctuation", "}"],
+
+	["punctuation", "}"]
 ]
 
 ----------------------------------------------------