From 10bce38a76d4d953854fadcca6d4c3a2645d8ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 8 Dec 2021 12:24:13 +0100 Subject: [PATCH 1/2] Update Elixir tokenization of sigil modifiers --- src/basic-languages/elixir/elixir.test.ts | 11 +++++++++++ src/basic-languages/elixir/elixir.ts | 13 +++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/basic-languages/elixir/elixir.test.ts b/src/basic-languages/elixir/elixir.test.ts index a6e9d26ff0..bd60257e55 100644 --- a/src/basic-languages/elixir/elixir.test.ts +++ b/src/basic-languages/elixir/elixir.test.ts @@ -314,6 +314,17 @@ testTokenization('elixir', [ ] } ], + // Sigils (modifiers) + [ + { + line: '~X/custom/az09', + tokens: [ + { startIndex: 0, type: 'sigil.delimiter.elixir' }, + { startIndex: 3, type: 'sigil.elixir' }, + { startIndex: 9, type: 'sigil.delimiter.elixir' } + ] + } + ], // Module attributes [ { diff --git a/src/basic-languages/elixir/elixir.ts b/src/basic-languages/elixir/elixir.ts index f294e7c346..4e76a674f9 100644 --- a/src/basic-languages/elixir/elixir.ts +++ b/src/basic-languages/elixir/elixir.ts @@ -125,6 +125,7 @@ export const language = { sigilSymmetricDelimiter: /"""|'''|"|'|\/|\|/, sigilStartDelimiter: /@sigilSymmetricDelimiter|<|\{|\[|\(/, sigilEndDelimiter: /@sigilSymmetricDelimiter|>|\}|\]|\)/, + sigilModifiers: /[a-zA-Z0-9]*/, decimal: /\d(?:_?\d)*/, hex: /[0-9a-fA-F](_?[0-9a-fA-F])*/, @@ -387,7 +388,7 @@ export const language = { 'sigilContinue.interpol.s': [ [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'string.delimiter', next: '@pop' }, @@ -412,7 +413,7 @@ export const language = { // Ignore escaped sigil end [/(^|[^\\])\\@sigilEndDelimiter/, 'string'], [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'string.delimiter', next: '@pop' }, @@ -435,7 +436,7 @@ export const language = { 'sigilContinue.interpol.r': [ [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'regexp.delimiter', next: '@pop' }, @@ -460,7 +461,7 @@ export const language = { // Ignore escaped sigil end [/(^|[^\\])\\@sigilEndDelimiter/, 'regexp'], [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'regexp.delimiter', next: '@pop' }, @@ -484,7 +485,7 @@ export const language = { 'sigilContinue.interpol': [ [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'sigil.delimiter', next: '@pop' }, @@ -509,7 +510,7 @@ export const language = { // Ignore escaped sigil end [/(^|[^\\])\\@sigilEndDelimiter/, 'sigil'], [ - /(@sigilEndDelimiter)[a-zA-Z]*/, + /(@sigilEndDelimiter)@sigilModifiers/, { cases: { '$1==$S5': { token: 'sigil.delimiter', next: '@pop' }, From 1126efe8d221939eb4311de232b421e14ca6d019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 8 Dec 2021 12:27:27 +0100 Subject: [PATCH 2/2] Tokenize the power operator --- src/basic-languages/elixir/elixir.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic-languages/elixir/elixir.ts b/src/basic-languages/elixir/elixir.ts index 4e76a674f9..ad974a200a 100644 --- a/src/basic-languages/elixir/elixir.ts +++ b/src/basic-languages/elixir/elixir.ts @@ -109,7 +109,7 @@ export const language = { // Matches any of the operator names: // <<< >>> ||| &&& ^^^ ~~~ === !== ~>> <~> |~> <|> == != <= >= && || \\ <> ++ -- |> =~ -> <- ~> <~ :: .. = < > + - * / | . ^ & ! operator: - /-[->]?|!={0,2}|\*|\/|\\\\|&{1,3}|\.\.?|\^(?:\^\^)?|\+\+?|<(?:-|<<|=|>|\|>|~>?)?|=~|={1,3}|>(?:=|>>)?|\|~>|\|>|\|{1,3}|~>>?|~~~|::/, + /-[->]?|!={0,2}|\*{1,2}|\/|\\\\|&{1,3}|\.\.?|\^(?:\^\^)?|\+\+?|<(?:-|<<|=|>|\|>|~>?)?|=~|={1,3}|>(?:=|>>)?|\|~>|\|>|\|{1,3}|~>>?|~~~|::/, // See https://hexdocs.pm/elixir/syntax-reference.html#variables variableName: /[a-z_][a-zA-Z0-9_]*[?!]?/,