Tolerate underscores in number literals

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to MiniJinja are documented here.
 
+## 1.0.16
+
+- Tolerate underscores in number literals.  #443
+
 ## 1.0.15
 
 - Resolved a compiler warning for Rust 1.77.  #440

minijinja/src/compiler/lexer.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
 use crate::compiler::tokens::{Span, Token};
 use crate::error::{Error, ErrorKind};
 use crate::utils::{memchr, memstr, unescape};
@@ -247,6 +249,7 @@ impl<'s> TokenizerState<'s> {
             .iter()
             .take_while(|&c| c.is_ascii_digit())
             .count();
+        let mut has_underscore = false;
         for c in self.rest.as_bytes()[num_len..].iter().copied() {
             state = match (c, state) {
                 (b'.', State::Integer) => State::Fraction,
@@ -255,22 +258,33 @@ impl<'s> TokenizerState<'s> {
                 (b'0'..=b'9', State::Exponent) => State::ExponentSign,
                 (b'0'..=b'9', state) => state,
                 (b'a'..=b'f' | b'A'..=b'F', State::RadixInteger) if radix == 16 => state,
+                (b'_', _) => {
+                    has_underscore = true;
+                    state
+                }
                 _ => break,
             };
             num_len += 1;
         }
         let is_float = !matches!(state, State::Integer | State::RadixInteger);
 
-        let num = self.advance(num_len);
+        let mut num = Cow::Borrowed(self.advance(num_len));
+        if has_underscore {
+            if num.ends_with('_') {
+                return Err(self.syntax_error("'_' may not occur at end of number"));
+            }
+            num = Cow::Owned(num.replace('_', ""));
+        }
+
         Ok((
             ok!(if is_float {
                 num.parse()
                     .map(Token::Float)
                     .map_err(|_| self.syntax_error("invalid float"))
-            } else if let Ok(int) = u64::from_str_radix(num, radix) {
+            } else if let Ok(int) = u64::from_str_radix(&num, radix) {
                 Ok(Token::Int(int))
             } else {
-                u128::from_str_radix(num, radix)
+                u128::from_str_radix(&num, radix)
                     .map(Token::Int128)
                     .map_err(|_| self.syntax_error("invalid integer"))
             }),

minijinja/src/syntax.rs

@@ -82,7 +82,9 @@
 //!   and filters, or just to extend or include a template).
 //! - `42`: Integers are whole numbers without a decimal part.  They can be prefixed with
 //!   `0b` to indicate binary, `0o` to indicate octal and `0x` to indicate hexadecimal.
+//!   Underscores are tolerated (and ignored) everywhere a digit is except in the last place.
 //! - `42.0`: Floating point numbers can be written using a `.` as a decimal mark.
+//!   Underscores are tolerated (and ignored) everywhere a digit is except in the last place.
 //! - `['list', 'of', 'objects']`: Everything between two brackets is a list. Lists are useful
 //!   for storing sequential data to be iterated over.
 //!   for compatibility with Jinja2 `('list', 'of', 'objects')` is also allowed.

minijinja/tests/inputs/err_bad_trailing_underscore.txt

@@ -0,0 +1,3 @@
+{}
+---
+{{ 10_ }}

minijinja/tests/lexer-inputs/literals.txt

@@ -2,16 +2,25 @@
 {{ false }}
 {{ null }}
 {{ 1.0 }}
+{{ 1_000.0 }}
 {{ 2 }}
+{{ 2_000 }}
 {{ [1, 2, 3] }}
 {{ {"foo": "bar"} }}
 {{ -9223372036854775808 }}
+{{ -9_223_372_036_854_775_808 }}
 {{ -170141183460469231731687303715884105728 }}
 {{ 170141183460469231731687303715884105727 }}
 {{ 340282366920938463463374607431768211455 }}
 {{ 0b1111 }}
 {{ 0B1001 }}
 {{ 0o777 }}
 {{ 0O777 }}
+{{ 0O7_77 }}
 {{ 0xdeadbeef }}
 {{ 0XDEADBEEF }}
+{{ 0XDEAD_BEEF }}
+{{ 1e2 }}
+{{ 1e_2 }}
+{{ 1_1e+_2 }}
+{{ 1_1e-_2 }}

minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap

@@ -1,6 +1,6 @@
 ---
 source: minijinja/tests/test_lexer.rs
-description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}"
+description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 1_000.0 }}\n{{ 2 }}\n{{ 2_000 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -9_223_372_036_854_775_808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0O7_77 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}\n{{ 0XDEAD_BEEF }}\n{{ 1e2 }}\n{{ 1e_2 }}\n{{ 1_1e+_2 }}\n{{ 1_1e-_2 }}"
 input_file: minijinja/tests/lexer-inputs/literals.txt
 ---
 VariableStart
@@ -37,6 +37,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Float(1000.0)
+  "1_000.0"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 Int(2)
   "2"
 VariableEnd
@@ -45,6 +53,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Int(2000)
+  "2_000"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 BracketOpen
   "["
 Int(1)
@@ -93,6 +109,16 @@ VariableStart
   "{{"
 Minus
   "-"
+Int(9223372036854775808)
+  "9_223_372_036_854_775_808"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Minus
+  "-"
 Int128(170141183460469231731687303715884105728)
   "170141183460469231731687303715884105728"
 VariableEnd
@@ -149,6 +175,14 @@ TemplateData("\n")
   "\n"
 VariableStart
   "{{"
+Int(511)
+  "0O7_77"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
 Int(3735928559)
   "0xdeadbeef"
 VariableEnd
@@ -163,3 +197,43 @@ VariableEnd
   "}}"
 TemplateData("\n")
   "\n"
+VariableStart
+  "{{"
+Int(3735928559)
+  "0XDEAD_BEEF"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Float(100.0)
+  "1e2"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Float(100.0)
+  "1e_2"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Float(1100.0)
+  "1_1e+_2"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Float(0.11)
+  "1_1e-_2"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"

minijinja/tests/snapshots/test_templates__vm@err_bad_trailing_underscore.txt.snap

@@ -0,0 +1,22 @@
+---
+source: minijinja/tests/test_templates.rs
+description: "{{ 10_ }}"
+info: {}
+input_file: minijinja/tests/inputs/err_bad_trailing_underscore.txt
+---
+!!!SYNTAX ERROR!!!
+
+Error {
+    kind: SyntaxError,
+    detail: "'_' may not occur at end of number",
+    name: "err_bad_trailing_underscore.txt",
+    line: 1,
+}
+
+syntax error: '_' may not occur at end of number (in err_bad_trailing_underscore.txt:1)
+----------------------- err_bad_trailing_underscore.txt -----------------------
+   1 > {{ 10_ }}
+     i ^^ syntax error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+No referenced variables
+-------------------------------------------------------------------------------