Tolerate underscores in number literals

mitsuhiko · Mar 22, 2024 · 746ea4f · 746ea4f
1 parent a1f9e28
commit 746ea4f
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 All notable changes to MiniJinja are documented here.
 
+## 1.0.15
+
+- Tolerate underscores in number literals.
+
 ## 1.0.14
 
 - Fixed a bug with broken closure handling when working with nested

diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
@@ -245,7 +245,7 @@ impl<'s> TokenizerState<'s> {
             .rest
             .as_bytes()
             .iter()
-            .take_while(|&c| c.is_ascii_digit())
+            .take_while(|&c| c.is_ascii_digit() || *c == b'_')
             .count();
         for c in self.rest.as_bytes()[num_len..].iter().copied() {
             state = match (c, state) {
@@ -255,22 +255,31 @@ impl<'s> TokenizerState<'s> {
                 (b'0'..=b'9', State::Exponent) => State::ExponentSign,
                 (b'0'..=b'9', state) => state,
                 (b'a'..=b'f' | b'A'..=b'F', State::RadixInteger) if radix == 16 => state,
+                (
+                    b'_',
+                    State::RadixInteger | State::Integer | State::Fraction | State::Exponent,
+                ) => state,
                 _ => break,
             };
             num_len += 1;
         }
         let is_float = !matches!(state, State::Integer | State::RadixInteger);
 
-        let num = self.advance(num_len);
+        if num_len > 0 && self.rest.as_bytes()[num_len - 1] == b'_' {
+            return Result::Err(self.syntax_error("'_' may not occur at end of number"));
+        }
+
+        let num = self.advance(num_len).replace('_', "");
+
         Ok((
             ok!(if is_float {
                 num.parse()
                     .map(Token::Float)
                     .map_err(|_| self.syntax_error("invalid float"))
-            } else if let Ok(int) = u64::from_str_radix(num, radix) {
+            } else if let Ok(int) = u64::from_str_radix(&num, radix) {
                 Ok(Token::Int(int))
             } else {
-                u128::from_str_radix(num, radix)
+                u128::from_str_radix(&num, radix)
                     .map(Token::Int128)
                     .map_err(|_| self.syntax_error("invalid integer"))
             }),

diff --git a/minijinja/src/syntax.rs b/minijinja/src/syntax.rs
@@ -82,7 +82,9 @@
 //!   and filters, or just to extend or include a template).
 //! - `42`: Integers are whole numbers without a decimal part.  They can be prefixed with
 //!   `0b` to indicate binary, `0o` to indicate octal and `0x` to indicate hexadecimal.
+//!   Underscores are tolerated (and ignored) everywhere a digit is except in the last place.
 //! - `42.0`: Floating point numbers can be written using a `.` as a decimal mark.
+//!   Underscores are tolerated (and ignored) everywhere a digit is except in the last place.
 //! - `['list', 'of', 'objects']`: Everything between two brackets is a list. Lists are useful
 //!   for storing sequential data to be iterated over.
 //!   for compatibility with Jinja2 `('list', 'of', 'objects')` is also allowed.

diff --git a/minijinja/tests/lexer-inputs/literals.txt b/minijinja/tests/lexer-inputs/literals.txt
@@ -2,16 +2,21 @@
 {{ false }}
 {{ null }}
 {{ 1.0 }}
+{{ 1_000.0 }}
 {{ 2 }}
+{{ 2_000 }}
 {{ [1, 2, 3] }}
 {{ {"foo": "bar"} }}
 {{ -9223372036854775808 }}
+{{ -9_223_372_036_854_775_808 }}
 {{ -170141183460469231731687303715884105728 }}
 {{ 170141183460469231731687303715884105727 }}
 {{ 340282366920938463463374607431768211455 }}
 {{ 0b1111 }}
 {{ 0B1001 }}
 {{ 0o777 }}
 {{ 0O777 }}
+{{ 0O7_77 }}
 {{ 0xdeadbeef }}
 {{ 0XDEADBEEF }}
+{{ 0XDEAD_BEEF }}
diff --git a/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap b/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap
@@ -1,6 +1,7 @@
 ---
 source: minijinja/tests/test_lexer.rs
-description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}"
+assertion_line: 24
+description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 1_000.0 }}\n{{ 2 }}\n{{ 2_000 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -9_223_372_036_854_775_808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0O7_77 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}\n{{ 0XDEAD_BEEF }}"
 input_file: minijinja/tests/lexer-inputs/literals.txt
 ---
 VariableStart
@@ -37,6 +38,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Float(1000.0)
+  "1_000.0"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 Int(2)
   "2"
 VariableEnd
@@ -45,6 +54,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Int(2000)
+  "2_000"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 BracketOpen
   "["
 Int(1)
@@ -93,6 +110,16 @@ VariableStart
   "{{"
 Minus
   "-"
+Int(9223372036854775808)
+  "9_223_372_036_854_775_808"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Minus
+  "-"
 Int128(170141183460469231731687303715884105728)
   "170141183460469231731687303715884105728"
 VariableEnd
@@ -149,6 +176,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Int(511)
+  "0O7_77"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 Int(3735928559)
   "0xdeadbeef"
 VariableEnd
@@ -163,3 +198,12 @@ VariableEnd
   "}}"
 TemplateData("\n")
   "\n"
+VariableStart
+  "{{"
+Int(3735928559)
+  "0XDEAD_BEEF"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+