Merge pull request #2806 from jonatanklosko/jk-elixir-sigils

Update Elixir tokenization of sigil modifiers
microsoft · Dec 8, 2021 · 86cff74 · 86cff74
2 parents 0afd248 + 1126efe
commit 86cff74
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 7 deletions.
diff --git a/src/basic-languages/elixir/elixir.test.ts b/src/basic-languages/elixir/elixir.test.ts
@@ -314,6 +314,17 @@ testTokenization('elixir', [
 			]
 		}
 	],
+	// Sigils (modifiers)
+	[
+		{
+			line: '~X/custom/az09',
+			tokens: [
+				{ startIndex: 0, type: 'sigil.delimiter.elixir' },
+				{ startIndex: 3, type: 'sigil.elixir' },
+				{ startIndex: 9, type: 'sigil.delimiter.elixir' }
+			]
+		}
+	],
 	// Module attributes
 	[
 		{

diff --git a/src/basic-languages/elixir/elixir.ts b/src/basic-languages/elixir/elixir.ts
@@ -109,7 +109,7 @@ export const language = <languages.IMonarchLanguage>{
 	// Matches any of the operator names:
 	// <<< >>> ||| &&& ^^^ ~~~ === !== ~>> <~> |~> <|> == != <= >= && || \\ <> ++ -- |> =~ -> <- ~> <~ :: .. = < > + - * / | . ^ & !
 	operator:
-		/-[->]?|!={0,2}|\*|\/|\\\\|&{1,3}|\.\.?|\^(?:\^\^)?|\+\+?|<(?:-|<<|=|>|\|>|~>?)?|=~|={1,3}|>(?:=|>>)?|\|~>|\|>|\|{1,3}|~>>?|~~~|::/,
+		/-[->]?|!={0,2}|\*{1,2}|\/|\\\\|&{1,3}|\.\.?|\^(?:\^\^)?|\+\+?|<(?:-|<<|=|>|\|>|~>?)?|=~|={1,3}|>(?:=|>>)?|\|~>|\|>|\|{1,3}|~>>?|~~~|::/,
 
 	// See https://hexdocs.pm/elixir/syntax-reference.html#variables
 	variableName: /[a-z_][a-zA-Z0-9_]*[?!]?/,
@@ -125,6 +125,7 @@ export const language = <languages.IMonarchLanguage>{
 	sigilSymmetricDelimiter: /"""|'''|"|'|\/|\|/,
 	sigilStartDelimiter: /@sigilSymmetricDelimiter|<|\{|\[|\(/,
 	sigilEndDelimiter: /@sigilSymmetricDelimiter|>|\}|\]|\)/,
+	sigilModifiers: /[a-zA-Z0-9]*/,
 
 	decimal: /\d(?:_?\d)*/,
 	hex: /[0-9a-fA-F](_?[0-9a-fA-F])*/,
@@ -387,7 +388,7 @@ export const language = <languages.IMonarchLanguage>{
 
 		'sigilContinue.interpol.s': [
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'string.delimiter', next: '@pop' },
@@ -412,7 +413,7 @@ export const language = <languages.IMonarchLanguage>{
 			// Ignore escaped sigil end
 			[/(^|[^\\])\\@sigilEndDelimiter/, 'string'],
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'string.delimiter', next: '@pop' },
@@ -435,7 +436,7 @@ export const language = <languages.IMonarchLanguage>{
 
 		'sigilContinue.interpol.r': [
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'regexp.delimiter', next: '@pop' },
@@ -460,7 +461,7 @@ export const language = <languages.IMonarchLanguage>{
 			// Ignore escaped sigil end
 			[/(^|[^\\])\\@sigilEndDelimiter/, 'regexp'],
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'regexp.delimiter', next: '@pop' },
@@ -484,7 +485,7 @@ export const language = <languages.IMonarchLanguage>{
 
 		'sigilContinue.interpol': [
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'sigil.delimiter', next: '@pop' },
@@ -509,7 +510,7 @@ export const language = <languages.IMonarchLanguage>{
 			// Ignore escaped sigil end
 			[/(^|[^\\])\\@sigilEndDelimiter/, 'sigil'],
 			[
-				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				/(@sigilEndDelimiter)@sigilModifiers/,
 				{
 					cases: {
 						'$1==$S5': { token: 'sigil.delimiter', next: '@pop' },