fix: add underscore parsing in numbers for haskell (#1020)

alecthomas · Nov 1, 2024 · 5e7b53e · 5e7b53e
1 parent 2e669a2
commit 5e7b53e
Show file tree

Hide file tree

Showing 3 changed files with 146 additions and 5 deletions.
diff --git a/lexers/embedded/haskell.xml b/lexers/embedded/haskell.xml
@@ -95,19 +95,22 @@
       <rule pattern="[:!#$%&amp;*+.\\/&lt;=&gt;?@^|~-]+">
         <token type="Operator"/>
       </rule>
-      <rule pattern="\d+[eE][+-]?\d+">
+      <rule pattern="\d+_*[eE][+-]?\d+">
         <token type="LiteralNumberFloat"/>
       </rule>
-      <rule pattern="\d+\.\d+([eE][+-]?\d+)?">
+      <rule pattern="\d+(_+[\d]+)*\.\d+(_+[\d]+)*([eE][+-]?\d+)?">
         <token type="LiteralNumberFloat"/>
       </rule>
-      <rule pattern="0[oO][0-7]+">
+      <rule pattern="0[oO](_*[0-7])+">
         <token type="LiteralNumberOct"/>
       </rule>
-      <rule pattern="0[xX][\da-fA-F]+">
+      <rule pattern="0[xX](_*[\da-fA-F])+">
         <token type="LiteralNumberHex"/>
       </rule>
-      <rule pattern="\d+">
+      <rule pattern="0[bB](_*[01])+">
+        <token type="LiteralNumberBin"/>
+      </rule>
+      <rule pattern="\d+(_*[\d])*">
         <token type="LiteralNumberInteger"/>
       </rule>
       <rule pattern="&#39;">

diff --git a/lexers/testdata/haskell.actual b/lexers/testdata/haskell.actual
@@ -0,0 +1,43 @@
+1_000_000
+1__000000
+1000000_
+_1000000
+
+0.0001
+0.000_1
+0_.0001
+_0.0001
+0._0001
+0.0001_
+
+1e+23
+1_e+23
+1__e+23
+1e_+23
+
+1e+23
+1e+_23
+1e+23_
+
+0x
+0xffff
+0xff_ff
+0x_ffff
+0x__ffff
+_0xffff
+
+0b
+0b01_0000__0000
+0b01_
+0b_01
+0b__01
+
+3.141_592_653_589_793
+96_485.332_89
+6.022_140_857e+23
+
+0o
+0o7_77
+0o700_
+0o_700
+0o__700
diff --git a/lexers/testdata/haskell.expected b/lexers/testdata/haskell.expected
@@ -0,0 +1,95 @@
+[
+  {"type":"LiteralNumberInteger","value":"1_000_000"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"1__000000"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"1000000"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameFunction","value":"_1000000"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberFloat","value":"0.0001"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"0.000_1"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Name","value":"_"},
+  {"type":"Operator","value":"."},
+  {"type":"LiteralNumberInteger","value":"0001"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameFunction","value":"_0"},
+  {"type":"Operator","value":"."},
+  {"type":"LiteralNumberInteger","value":"0001"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Operator","value":"."},
+  {"type":"Name","value":"_0001"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"0.0001"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberFloat","value":"1e+23"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"1_e+23"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"1__e+23"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Name","value":"e_"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"23"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberFloat","value":"1e+23"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Name","value":"e"},
+  {"type":"Operator","value":"+"},
+  {"type":"Name","value":"_23"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"1e+23"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Name","value":"x"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberHex","value":"0xffff"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberHex","value":"0xff_ff"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberHex","value":"0x_ffff"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberHex","value":"0x__ffff"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameFunction","value":"_0xffff"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberBin","value":"0b01_0000__0000"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberBin","value":"0b01"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberBin","value":"0b_01"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberBin","value":"0b__01"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberFloat","value":"3.141_592_653_589_793"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"96_485.332_89"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberFloat","value":"6.022_140_857e+23"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Name","value":"o"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberOct","value":"0o7_77"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberOct","value":"0o700"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberOct","value":"0o_700"},
+  {"type":"Text","value":"\n"},
+  {"type":"LiteralNumberOct","value":"0o__700"},
+  {"type":"Text","value":"\n"}
+]